01HMYBQAM126KRYHT7QJ7C68PB: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HMYBQQ3E4SST4EGSM4XNKM4R

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-01-24T18:58:54.285Zjob dependencies complete; ready to run (waiting for 29 m 33 s)
22024-01-24T19:03:40.785Zjob assigned to worker 01HMYDMQ3PDT27H46C3B0F0G35 (queued for 4 m 46 s)
32024-01-24T19:03:44.380Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-01-24T19:03:56.941Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-01-24T19:03:56.941Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-01-24T19:04:07.952Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-01-24T19:04:07.952Zdownloading input: /input/build/work/bins/crutest.gz
82024-01-24T19:04:21.631Zdownloaded input: /input/build/work/bins/crutest.gz
92024-01-24T19:04:21.631Zdownloading input: /input/build/work/bins/dsc.gz
102024-01-24T19:04:36.388Zdownloaded input: /input/build/work/bins/dsc.gz
112024-01-24T19:04:36.430Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-01-24T19:04:36.499Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-01-24T19:04:36.499Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-01-24T19:04:36.513Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-01-24T19:04:36.513Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-01-24T19:04:37.011Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-01-24T19:04:37.011Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-01-24T19:04:37.052Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-01-24T19:04:37.052Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-01-24T19:04:37.075Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-01-24T19:04:37.075Zdownloading input: /input/build/work/scripts/test_up.sh
222024-01-24T19:04:37.513Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-01-24T19:04:37.513Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-01-24T19:04:37.532Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-01-24T19:04:37.538Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-01-24T19:05:04.952Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-01-24T19:05:04.952Zstarting task 0: "setup"
282024-01-24T19:05:05.056Z++ uname -s
292024-01-24T19:05:05.066Z+ kern=SunOS
302024-01-24T19:05:05.066Z+ case "$kern" in
312024-01-24T19:05:05.066Z+ groupadd -g 12345 build
322024-01-24T19:05:05.066Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-01-24T19:05:07.589Z+ zfs create -o mountpoint=/work rpool/work
342024-01-24T19:05:07.602Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-01-24T19:05:07.604Z+ home_fs=zfs
362024-01-24T19:05:07.604Z+ [[ zfs == autofs ]]
372024-01-24T19:05:07.605Z+ mkdir -p /home/build
382024-01-24T19:05:07.605Z+ chown build:build /home/build /work
392024-01-24T19:05:09.310Z+ chmod 0700 /home/build /work
402024-01-24T19:05:09.325Zprocess exited: duration 4405 ms, exit code 0
 
412024-01-24T19:05:09.766Zstarting task 1: "authentication"
422024-01-24T19:05:09.796Zprocess exited: duration 33 ms, exit code 0
 
432024-01-24T19:05:10.229Zstarting task 2: "build"
442024-01-24T19:05:10.237Z+ banner cores
452024-01-24T19:05:10.350Z
462024-01-24T19:05:10.350Z #### #### ##### ###### ####
472024-01-24T19:05:10.350Z # # # # # # # #
482024-01-24T19:05:10.350Z # # # # # ##### ####
492024-01-24T19:05:10.350Z # # # ##### # #
502024-01-24T19:05:10.350Z # # # # # # # # #
512024-01-24T19:05:10.350Z #### #### # # ###### ####
522024-01-24T19:05:10.351Z
532024-01-24T19:05:10.391Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-01-24T19:05:10.391Z+ echo 'input bins dir contains:'
552024-01-24T19:05:10.391Zinput bins dir contains:
562024-01-24T19:05:10.392Z+ ls -ltr /input/build/work/bins
572024-01-24T19:05:10.392Ztotal 710919
582024-01-24T19:05:10.392Z-rw-r--r-- 1 root root 110577064 Jan 24 19:03 crucible-downstairs.gz
592024-01-24T19:05:10.393Z-rw-r--r-- 1 root root 87230459 Jan 24 19:04 crucible-hammer.gz
602024-01-24T19:05:10.394Z-rw-r--r-- 1 root root 96248494 Jan 24 19:04 crutest.gz
612024-01-24T19:05:10.394Z-rw-r--r-- 1 root root 69700623 Jan 24 19:04 dsc.gz
622024-01-24T19:05:10.394Z+ echo 'input script dir contains:'
632024-01-24T19:05:10.394Zinput script dir contains:
642024-01-24T19:05:10.394Z+ ls -ltr /input/build/work/scripts
652024-01-24T19:05:10.395Ztotal 38
662024-01-24T19:05:10.395Z-rw-r--r-- 1 root root 3056 Jan 24 19:04 perf-downstairs-tick.d
672024-01-24T19:05:10.395Z-rw-r--r-- 1 root root 939 Jan 24 19:04 test_ds.sh
682024-01-24T19:05:10.395Z-rw-r--r-- 1 root root 3746 Jan 24 19:04 test_live_repair.sh
692024-01-24T19:05:10.395Z-rw-r--r-- 1 root root 6416 Jan 24 19:04 test_repair.sh
702024-01-24T19:05:10.395Z-rw-r--r-- 1 root root 2969 Jan 24 19:04 test_replay.sh
712024-01-24T19:05:10.396Z-rw-r--r-- 1 root root 9465 Jan 24 19:04 test_up.sh
722024-01-24T19:05:10.396Z-rw-r--r-- 1 root root 2824 Jan 24 19:04 upstairs_info.d
732024-01-24T19:05:10.396Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-01-24T19:05:10.396Z+ echo ' chmod input script dir contains:'
752024-01-24T19:05:10.396Z chmod input script dir contains:
762024-01-24T19:05:10.396Z+ ls -ltr /input/build/work/scripts
772024-01-24T19:05:10.397Ztotal 38
782024-01-24T19:05:10.397Z-rwxr-xr-x 1 root root 3056 Jan 24 19:04 perf-downstairs-tick.d
792024-01-24T19:05:10.397Z-rwxr-xr-x 1 root root 939 Jan 24 19:04 test_ds.sh
802024-01-24T19:05:10.397Z-rwxr-xr-x 1 root root 3746 Jan 24 19:04 test_live_repair.sh
812024-01-24T19:05:10.397Z-rwxr-xr-x 1 root root 6416 Jan 24 19:04 test_repair.sh
822024-01-24T19:05:10.398Z-rwxr-xr-x 1 root root 2969 Jan 24 19:04 test_replay.sh
832024-01-24T19:05:10.398Z-rwxr-xr-x 1 root root 9465 Jan 24 19:04 test_up.sh
842024-01-24T19:05:10.399Z-rwxr-xr-x 1 root root 2824 Jan 24 19:04 upstairs_info.d
852024-01-24T19:05:10.399Z+ banner unpack
862024-01-24T19:05:10.399Z
872024-01-24T19:05:10.399Z # # # # ##### ## #### # #
882024-01-24T19:05:10.399Z # # ## # # # # # # # # #
892024-01-24T19:05:10.399Z # # # # # # # # # # ####
902024-01-24T19:05:10.400Z # # # # # ##### ###### # # #
912024-01-24T19:05:10.400Z # # # ## # # # # # # #
922024-01-24T19:05:10.400Z #### # # # # # #### # #
932024-01-24T19:05:10.400Z
942024-01-24T19:05:10.400Z+ mkdir -p /var/tmp/bins
952024-01-24T19:05:10.400Z+ for t in "$input/bins/"*.gz
962024-01-24T19:05:10.401Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-01-24T19:05:10.401Z+ b=crucible-downstairs.gz
982024-01-24T19:05:10.401Z+ b=crucible-downstairs
992024-01-24T19:05:10.401Z+ gunzip
1002024-01-24T19:05:14.335Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-01-24T19:05:14.343Z+ for t in "$input/bins/"*.gz
1022024-01-24T19:05:14.343Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-01-24T19:05:14.343Z+ b=crucible-hammer.gz
1042024-01-24T19:05:14.343Z+ b=crucible-hammer
1052024-01-24T19:05:14.343Z+ gunzip
1062024-01-24T19:05:17.403Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-01-24T19:05:17.423Z+ for t in "$input/bins/"*.gz
1082024-01-24T19:05:17.424Z++ basename /input/build/work/bins/crutest.gz
1092024-01-24T19:05:17.426Z+ b=crutest.gz
1102024-01-24T19:05:17.426Z+ b=crutest
1112024-01-24T19:05:17.427Z+ gunzip
1122024-01-24T19:05:20.655Z+ chmod +x /var/tmp/bins/crutest
1132024-01-24T19:05:20.675Z+ for t in "$input/bins/"*.gz
1142024-01-24T19:05:20.675Z++ basename /input/build/work/bins/dsc.gz
1152024-01-24T19:05:20.675Z+ b=dsc.gz
1162024-01-24T19:05:20.675Z+ b=dsc
1172024-01-24T19:05:20.675Z+ gunzip
1182024-01-24T19:05:23.194Z+ chmod +x /var/tmp/bins/dsc
1192024-01-24T19:05:23.215Z+ export BINDIR=/var/tmp/bins
1202024-01-24T19:05:23.216Z+ BINDIR=/var/tmp/bins
1212024-01-24T19:05:23.216Z+ banner repair
1222024-01-24T19:05:23.216Z
1232024-01-24T19:05:23.216Z ##### ###### ##### ## # #####
1242024-01-24T19:05:23.216Z # # # # # # # # # #
1252024-01-24T19:05:23.216Z # # ##### # # # # # # #
1262024-01-24T19:05:23.216Z ##### # ##### ###### # #####
1272024-01-24T19:05:23.217Z # # # # # # # # #
1282024-01-24T19:05:23.217Z # # ###### # # # # # #
1292024-01-24T19:05:23.217Z
1302024-01-24T19:05:23.217Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-01-24T19:05:23.334ZTurn off color for downstairs dump
1322024-01-24T19:05:23.365ZCreating region directory at: /var/tmp/test_repair
1332024-01-24T19:05:23.379ZCreating dsc directory at: /tmp/dsc
1342024-01-24T19:05:23.790ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.42604867
1352024-01-24T19:05:23.844ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.06143323
1362024-01-24T19:05:23.903ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.056866575
1372024-01-24T19:05:23.903ZRegion set with 3 regions was created
1382024-01-24T19:05:23.918Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1131 1132 1133
1392024-01-24T19:05:23.918ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-01-24T19:05:23.974Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.617242474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135}
1412024-01-24T19:05:24.436Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.618289392Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135}
1422024-01-24T19:05:24.436Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.618346014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135}
1432024-01-24T19:05:24.436Z{"msg":"Crucible b30b8325-981b-4fbf-942c-e38a73d8e23d has session id: e50f4704-f66e-40f0-8a1a-0f68bd507e79","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.618372674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1442024-01-24T19:05:24.436Z{"msg":"Upstairs opts: Upstairs UUID: b30b8325-981b-4fbf-942c-e38a73d8e23d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.618438319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1452024-01-24T19:05:24.438Z{"msg":"Crucible stats registered with UUID: b30b8325-981b-4fbf-942c-e38a73d8e23d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.61850196Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1462024-01-24T19:05:24.438ZCrucible runtime is spawned
1472024-01-24T19:05:24.438Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.620414825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135}
1482024-01-24T19:05:24.439Z{"msg":"b30b8325-981b-4fbf-942c-e38a73d8e23d active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.620816539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1492024-01-24T19:05:24.439Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.620931791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1502024-01-24T19:05:24.439Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621025155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1512024-01-24T19:05:24.439Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621085948Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1522024-01-24T19:05:24.439Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621130734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1532024-01-24T19:05:24.439Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621198968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1542024-01-24T19:05:24.439Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621244082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1552024-01-24T19:05:24.439Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621300137Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1562024-01-24T19:05:24.439Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621344119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1572024-01-24T19:05:24.439Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621386077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1582024-01-24T19:05:24.439Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621660285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1592024-01-24T19:05:24.439Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621752079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1602024-01-24T19:05:24.439Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.621800373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1612024-01-24T19:05:24.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.62618876Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1622024-01-24T19:05:24.440Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.626254831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1632024-01-24T19:05:24.440Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.626282885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1642024-01-24T19:05:24.440Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.626305212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1652024-01-24T19:05:24.441Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.626326721Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1662024-01-24T19:05:24.441Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.626396366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1672024-01-24T19:05:24.442Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.629923265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1682024-01-24T19:05:24.442Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.630224955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1692024-01-24T19:05:24.442Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.630283304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1702024-01-24T19:05:24.442Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.631663965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1712024-01-24T19:05:24.442Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.631746888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1722024-01-24T19:05:24.442Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.631790015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1732024-01-24T19:05:24.442Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.63185845Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1742024-01-24T19:05:24.442Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.631933805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1752024-01-24T19:05:24.442Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.631962934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1762024-01-24T19:05:24.442Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632004907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1772024-01-24T19:05:24.443Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632030149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1782024-01-24T19:05:24.443Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632074107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1792024-01-24T19:05:24.443Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632100228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1802024-01-24T19:05:24.443Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.63254917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1812024-01-24T19:05:24.443Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632688062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1822024-01-24T19:05:24.443Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.63274386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1832024-01-24T19:05:24.443Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632771988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1842024-01-24T19:05:24.443Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632817459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1852024-01-24T19:05:24.443Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632842971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1862024-01-24T19:05:24.443Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632887862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1872024-01-24T19:05:24.443Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.63291297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1882024-01-24T19:05:24.443Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632971581Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1892024-01-24T19:05:24.444Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.632996051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1902024-01-24T19:05:24.444Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633483288Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1912024-01-24T19:05:24.444Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633550116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1922024-01-24T19:05:24.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1932024-01-24T19:05:24.445Z,"time":"2024-01-24T19:05:22.633574486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1942024-01-24T19:05:24.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633603797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1952024-01-24T19:05:24.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633625884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1962024-01-24T19:05:24.447Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633670194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1972024-01-24T19:05:24.447Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633693962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1982024-01-24T19:05:24.447Z{"msg":"b30b8325-981b-4fbf-942c-e38a73d8e23d is now active with session: e50f4704-f66e-40f0-8a1a-0f68bd507e79","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633715365Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
1992024-01-24T19:05:24.447Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633760324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135}
2002024-01-24T19:05:24.447Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:22.633781493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1135,"session_id":"e50f4704-f66e-40f0-8a1a-0f68bd507e79"}
2012024-01-24T19:05:24.447ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-01-24T19:05:24.447ZFill test
2032024-01-24T19:05:24.474ZRead and Verify all blocks (0..600 range:false)
2042024-01-24T19:05:24.530ZWrote out file "/tmp/test_repair_verify.data"
2052024-01-24T19:05:24.531ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-01-24T19:05:24.539Z----------------------------------------------------------------
2072024-01-24T19:05:24.539Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:2
2082024-01-24T19:05:24.539ZStates: Active Active Active
2092024-01-24T19:05:24.539ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-01-24T19:05:24.539Z 13 Acked 1012 Read 100 Done Done Done false
2112024-01-24T19:05:24.539Z 14 Acked 1013 Read 100 Done Done Done false
2122024-01-24T19:05:24.539Z STATES DS:0 DS:1 DS:2 TOTAL
2132024-01-24T19:05:24.539Z New 0 0 0 0
2142024-01-24T19:05:24.539Z Sent 0 0 0 0
2152024-01-24T19:05:24.539Z Done 2 2 2 6
2162024-01-24T19:05:24.540Z Skipped 0 0 0 0
2172024-01-24T19:05:24.540Z Error 0 0 0 0
2182024-01-24T19:05:24.540ZLast Flush: 1011 1011 1011
2192024-01-24T19:05:24.540ZDownstairs last five completed: 1011 1010 1009 1008 1007
2202024-01-24T19:05:24.540ZUpstairs last five completed: 14 13 12 11 10
2212024-01-24T19:05:24.540ZCLIENT: Up:0 ds:2 act:3
2222024-01-24T19:05:28.538Z----------------------------------------------------------------
2232024-01-24T19:05:28.539Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2242024-01-24T19:05:28.539ZDownstairs last five completed: 1014 1013 1012 1011 1010
2252024-01-24T19:05:28.539ZUpstairs last five completed: 15 14 13 12 11
2262024-01-24T19:05:28.539ZCLIENT: Up:0 ds:0 act:3
2272024-01-24T19:05:28.539ZCLIENT: All crucible jobs finished, exiting program
2282024-01-24T19:05:28.555Z
2292024-01-24T19:05:28.555ZBegin loop 1
2302024-01-24T19:05:28.556ZDownstairs to restart: 0
2312024-01-24T19:05:29.013Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.224590299Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137}
2322024-01-24T19:05:29.013Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.224999224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137}
2332024-01-24T19:05:29.013Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225060685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137}
2342024-01-24T19:05:29.014Z{"msg":"Crucible e504321e-71c1-4ff8-b2a4-ca3362d2bb25 has session id: d7496de0-1301-446d-8953-1af81b8db048","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225085658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2352024-01-24T19:05:29.014Z{"msg":"Upstairs opts: Upstairs UUID: e504321e-71c1-4ff8-b2a4-ca3362d2bb25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225161612Z","Crucible runtime is spawned
2362024-01-24T19:05:29.014Zhostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2372024-01-24T19:05:29.014Z{"msg":"Crucible stats registered with UUID: e504321e-71c1-4ff8-b2a4-ca3362d2bb25","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225305535Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2382024-01-24T19:05:29.014Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225361097Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137}
2392024-01-24T19:05:29.016Z{"msg":"e504321e-71c1-4ff8-b2a4-ca3362d2bb25 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225633624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2402024-01-24T19:05:29.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225731032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2412024-01-24T19:05:29.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225822166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2422024-01-24T19:05:29.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.225922166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2432024-01-24T19:05:29.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226266756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2442024-01-24T19:05:29.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226316897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2452024-01-24T19:05:29.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226387541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2462024-01-24T19:05:29.017Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226438789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2472024-01-24T19:05:29.017Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226532941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2482024-01-24T19:05:29.017Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226576961Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2492024-01-24T19:05:29.018Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226618467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2502024-01-24T19:05:29.018Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226696235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2512024-01-24T19:05:29.018Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.226740593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2522024-01-24T19:05:29.019Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.227541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2532024-01-24T19:05:29.019Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.227605548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2542024-01-24T19:05:29.019Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.227708033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2552024-01-24T19:05:29.019Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.227766594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2562024-01-24T19:05:29.019Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.22788171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2572024-01-24T19:05:29.020Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.227937396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2582024-01-24T19:05:29.020Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.228632689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2592024-01-24T19:05:29.020Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.228808817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2602024-01-24T19:05:29.020Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.228914442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2612024-01-24T19:05:29.020Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229413253Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2622024-01-24T19:05:29.020Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229459414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2632024-01-24T19:05:29.020Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229501741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2642024-01-24T19:05:29.020Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229565653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2652024-01-24T19:05:29.020Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229611039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2662024-01-24T19:05:29.020Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229667863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2672024-01-24T19:05:29.020Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229859064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2682024-01-24T19:05:29.020Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.229957435Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2692024-01-24T19:05:29.021Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230037574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2702024-01-24T19:05:29.021Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.2300899Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2712024-01-24T19:05:29.021Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230117933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2722024-01-24T19:05:29.021Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
2732024-01-24T19:05:29.021Z30,"time":"2024-01-24T19:05:27.230165824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2742024-01-24T19:05:29.021Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230192858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2752024-01-24T19:05:29.021Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230248457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2762024-01-24T19:05:29.021Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.23027425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2772024-01-24T19:05:29.021Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230322829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2782024-01-24T19:05:29.021Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230349586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2792024-01-24T19:05:29.022Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230400553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2802024-01-24T19:05:29.022Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.23042595Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2812024-01-24T19:05:29.023Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.23044708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2822024-01-24T19:05:29.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230496869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2832024-01-24T19:05:29.023Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230521961Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2842024-01-24T19:05:29.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230541877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2852024-01-24T19:05:29.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230563934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2862024-01-24T19:05:29.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230642424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2872024-01-24T19:05:29.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230668816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2882024-01-24T19:05:29.024Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230695992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2892024-01-24T19:05:29.026Z{"msg":"e504321e-71c1-4ff8-b2a4-ca3362d2bb25 is now active with session: d7496de0-1301-446d-8953-1af81b8db048","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230764791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2902024-01-24T19:05:29.026Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2912024-01-24T19:05:29.026Z,"time":"2024-01-24T19:05:27.230789173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137,"session_id":"d7496de0-1301-446d-8953-1af81b8db048"}
2922024-01-24T19:05:29.026Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:27.230813875Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1137}
2932024-01-24T19:05:29.027ZLoading write count information from file "/tmp/test_repair_verify.data"
2942024-01-24T19:05:29.027ZRun Repair workload
2952024-01-24T19:05:29.027Z01/30 Write block 141 len 40960 data: 2 2 2 2 2 2 2 2 2 2
2962024-01-24T19:05:29.027Z02/30 Read block 551 len 24576
2972024-01-24T19:05:29.027Z03/30 Read block 250 len 12288
2982024-01-24T19:05:29.028Z04/30 Write block 96 len 20480 data: 2 2 2 2 2
2992024-01-24T19:05:29.028Z05/30 Read block 42 len 40960
3002024-01-24T19:05:29.028Z06/30 Write block 207 len 36864 data: 2 2 2 2 2 2 2 2 2
3012024-01-24T19:05:29.028Z07/30 Read block 472 len 24576
3022024-01-24T19:05:29.028Z08/30 Read block 212 len 32768
3032024-01-24T19:05:29.028Z09/30 Read block 278 len 32768
3042024-01-24T19:05:29.028Z10/30 Read block 143 len 8192
3052024-01-24T19:05:29.028Z11/30 Write block 323 len 28672 data: 2 2 2 2 2 2 2
3062024-01-24T19:05:29.028Z12/30 Read block 252 len 40960
3072024-01-24T19:05:29.028Z13/30 Write block 413 len 28672 data: 2 2 2 2 2 2 2
3082024-01-24T19:05:29.029Z14/30 Read block 268 len 16384
3092024-01-24T19:05:29.030Z15/30 Write block 510 len 24576 data: 2 2 2 2 2 2
3102024-01-24T19:05:29.030Z16/30 Write block 429 len 36864 data: 2 2 2 2 2 2 2 2 2
3112024-01-24T19:05:29.031Z17/30 Write block 433 len 12288 data: 3 3 3
3122024-01-24T19:05:29.031Z18/30 Write block 243 len 28672 data: 2 2 2 2 2 2 2
3132024-01-24T19:05:29.031Z19/30 Write block 224 len 16384 data: 2 2 2 2
3142024-01-24T19:05:29.031Z20/30 Write block 423 len 40960 data: 2 2 2 2 2 2 3 3 3 3
3152024-01-24T19:05:29.032Z21/30 Read block 173 len 8192
3162024-01-24T19:05:29.032Z22/30 Read block 478 len 16384
3172024-01-24T19:05:29.032Z23/30 Write block 517 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3182024-01-24T19:05:29.032Z24/30 Flush
3192024-01-24T19:05:29.033Z25/30 Write block 332 len 24576 data: 2 2 2 2 2 2
3202024-01-24T19:05:29.033Z26/30 Write block 24 len 24576 data: 2 2 2 2 2 2
3212024-01-24T19:05:29.033Z27/30 Write block 498 len 20480 data: 2 2 2 2 2
3222024-01-24T19:05:29.033Z28/30 Read block 437 len 24576
3232024-01-24T19:05:29.034Z29/30 Read block 598 len 4096
3242024-01-24T19:05:29.034Z30/30 Read block 70 len 20480
3252024-01-24T19:05:29.034Z----------------------------------------------------------------
3262024-01-24T19:05:29.034Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3272024-01-24T19:05:29.034ZStates: Active Active Active
3282024-01-24T19:05:29.034ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3292024-01-24T19:05:29.034Z 1 Acked 1000 Write 10 Done Done Done false
3302024-01-24T19:05:29.035Z 2 Acked 1001 Read 6 Done Done Done false
3312024-01-24T19:05:29.468Z 3 Acked 1002 Read 3 Done Done Done false
3322024-01-24T19:05:29.468Z 4 Acked 1003 Write 5 Done Done Done false
3332024-01-24T19:05:29.468Z 5 Acked 1004 Read 10 Sent Done Done false
3342024-01-24T19:05:29.468Z 6 Acked 1005 Write 9 Sent Done Done false
3352024-01-24T19:05:29.468Z 7 Acked 1006 Read 6 Sent Done Done false
3362024-01-24T19:05:29.468Z 8 Acked 1007 Read 8 Sent Done Done false
3372024-01-24T19:05:29.468Z 9 Acked 1008 Read 8 Sent Done Done false
3382024-01-24T19:05:29.468Z 10 Acked 1009 Read 2 Sent Done Done false
3392024-01-24T19:05:29.468Z 11 Acked 1010 Write 7 Sent Done Done false
3402024-01-24T19:05:29.469Z 12 Acked 1011 Read 10 Sent Done Done false
3412024-01-24T19:05:29.469Z 13 Acked 1012 Write 7 Sent Done Done false
3422024-01-24T19:05:29.469Z 14 Acked 1013 Read 4 Sent Done Done false
3432024-01-24T19:05:29.469Z 15 Acked 1014 Write 6 Sent Done Done false
3442024-01-24T19:05:29.469Z 16 Acked 1015 Write 9 Sent Done Done false
3452024-01-24T19:05:29.469Z 17 Acked 1016 Write 3 Sent Done Done false
3462024-01-24T19:05:29.469Z 18 Acked 1017 Write 7 Sent Done Done false
3472024-01-24T19:05:29.469Z 19 Acked 1018 Write 4 Sent Done Done false
3482024-01-24T19:05:29.469Z 20 Acked 1019 Write 10 Sent Done Done false
3492024-01-24T19:05:29.469Z 21 Acked 1020 Read 2 Sent Done Done false
3502024-01-24T19:05:29.469Z 22 Acked 1021 Read 4 Sent Done Done false
3512024-01-24T19:05:29.469Z 23 Acked 1022 Write 10 Sent Done Done false
3522024-01-24T19:05:29.469Z 24 Acked 1023 Flush 0 Sent Done Done false
3532024-01-24T19:05:29.469Z 25 Acked 1024 Write 6 Sent Done Done false
3542024-01-24T19:05:29.473Z 26 Acked 1025 Write 6 Sent Done Done false
3552024-01-24T19:05:29.473Z 27 Acked 1026 Write 5 Sent Done Done false
3562024-01-24T19:05:29.473Z 28 Acked 1027 Read 6 Sent Done Done false
3572024-01-24T19:05:29.473Z 29 Acked 1028 Read 1 Sent Done Done false
3582024-01-24T19:05:29.473Z 30 Acked 1029 Read 5 Sent Sent Done false
3592024-01-24T19:05:29.473Z STATES DS:0 DS:1 DS:2 TOTAL
3602024-01-24T19:05:29.473Z New 0 0 0 0
3612024-01-24T19:05:29.473Z Sent 26 1 0 27
3622024-01-24T19:05:29.473Z Done 4 29 30 63
3632024-01-24T19:05:29.474Z Skipped 0 0 0 0
3642024-01-24T19:05:29.474Z Error 0 0 0 0
3652024-01-24T19:05:29.474ZLast Flush: 0 1023 1023
3662024-01-24T19:05:29.474ZDownstairs last five completed:
3672024-01-24T19:05:29.474ZUpstairs last five completed: 30 29 28 27 26
3682024-01-24T19:05:29.474ZWrote out file "/tmp/test_repair_verify.data"
3692024-01-24T19:05:29.474Z
3702024-01-24T19:05:29.474ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3712024-01-24T19:05:29.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3722024-01-24T19:05:29.475Z 0 000-019 1 1 1 1 1 1 F F F
3732024-01-24T19:05:29.475Z 1 020-039 1 1 1 1 1 1 F T T <---
3742024-01-24T19:05:29.475Z 2 040-059 1 1 1 1 1 1 F F F
3752024-01-24T19:05:29.476Z 3 060-079 1 1 1 1 1 1 F F F
3762024-01-24T19:05:29.476Z 4 080-099 1 2 2 1 2 2 T F F <---
3772024-01-24T19:05:29.476Z 5 100-119 1 2 2 1 2 2 T F F <---
3782024-01-24T19:05:29.476Z 6 120-139 1 1 1 1 1 1 F F F
3792024-01-24T19:05:29.476Z 7 140-159 1 2 2 1 2 2 T F F <---
3802024-01-24T19:05:29.476Z 8 160-179 1 1 1 1 1 1 F F F
3812024-01-24T19:05:29.476Z 9 180-199 1 1 1 1 1 1 F F F
3822024-01-24T19:05:29.477Z 10 200-219 1 2 2 1 2 2 F F F <---
3832024-01-24T19:05:29.477Z 11 220-239 1 2 2 1 2 2 F F F <---
3842024-01-24T19:05:29.477Z 12 240-259 1 2 2 1 2 2 F F F <---
3852024-01-24T19:05:29.477Z 13 260-279 1 1 1 1 1 1 F F F
3862024-01-24T19:05:29.477Z 14 280-299 1 1 1 1 1 1 F F F
3872024-01-24T19:05:29.477Z 15 300-319 1 1 1 1 1 1 F F F
3882024-01-24T19:05:29.477Z 16 320-339 1 2 2 1 2 2 F T T <---
3892024-01-24T19:05:29.477Z 17 340-359 1 1 1 1 1 1 F F F
3902024-01-24T19:05:29.478Z 18 360-379 1 1 1 1 1 1 F F F
3912024-01-24T19:05:29.478Z 19 380-399 1 1 1 1 1 1 F F F
3922024-01-24T19:05:29.478Z 20 400-419 1 2 2 1 2 2 F F F <---
3932024-01-24T19:05:29.478Z 21 420-439 1 2 2 1 2 2 F F F <---
3942024-01-24T19:05:29.478Z 22 440-459 1 1 1 1 1 1 F F F
3952024-01-24T19:05:29.478Z 23 460-479 1 1 1 1 1 1 F F F
3962024-01-24T19:05:29.478Z 24 480-499 1 1 1 1 1 1 F T T <---
3972024-01-24T19:05:29.478Z 25 500-519 1 2 2 1 2 2 F T T <---
3982024-01-24T19:05:29.478Z 26 520-539 1 2 2 1 2 2 F F F <---
3992024-01-24T19:05:29.479Z 27 540-559 1 1 1 1 1 1 F F F
4002024-01-24T19:05:29.479Z 28 560-579 1 1 1 1 1 1 F F F
4012024-01-24T19:05:29.479Z 29 580-599 1 1 1 1 1 1 F F F
4022024-01-24T19:05:29.479ZMax gen: 2, Max flush: 2
4032024-01-24T19:05:29.479ZError: Difference in extent metadata found!
4042024-01-24T19:05:29.479ZA Difference in extent metadata is expected here
4052024-01-24T19:05:29.479ZOn loop 1
4062024-01-24T19:05:29.479Z
4072024-01-24T19:05:29.480ZVerifying data now
4082024-01-24T19:05:34.902ZNo change after verify
4092024-01-24T19:05:34.902ZLoop: 1 Downstairs dump after verify (and repair):
4102024-01-24T19:05:34.917ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4112024-01-24T19:05:34.917Z 0 000-019 1 1 1 1 1 1 F F F
4122024-01-24T19:05:34.917Z 1 020-039 3 3 3 3 3 3 F F F
4132024-01-24T19:05:34.917Z 2 040-059 1 1 1 1 1 1 F F F
4142024-01-24T19:05:34.917Z 3 060-079 1 1 1 1 1 1 F F F
4152024-01-24T19:05:34.917Z 4 080-099 2 2 2 2 2 2 F F F
4162024-01-24T19:05:34.918Z 5 100-119 2 2 2 2 2 2 F F F
4172024-01-24T19:05:34.919Z 6 120-139 1 1 1 1 1 1 F F F
4182024-01-24T19:05:34.919Z 7 140-159 2 2 2 2 2 2 F F F
4192024-01-24T19:05:34.919Z 8 160-179 1 1 1 1 1 1 F F F
4202024-01-24T19:05:34.929Z 9 180-199 1 1 1 1 1 1 F F F
4212024-01-24T19:05:34.930Z 10 200-219 2 2 2 2 2 2 F F F
4222024-01-24T19:05:34.930Z 11 220-239 2 2 2 2 2 2 F F F
4232024-01-24T19:05:34.930Z 12 240-259 2 2 2 2 2 2 F F F
4242024-01-24T19:05:34.931Z 13 260-279 1 1 1 1 1 1 F F F
4252024-01-24T19:05:34.934Z 14 280-299 1 1 1 1 1 1 F F F
4262024-01-24T19:05:34.935Z 15 300-319 1 1 1 1 1 1 F F F
4272024-01-24T19:05:34.935Z 16 320-339 3 3 3 3 3 3 F F F
4282024-01-24T19:05:34.935Z 17 340-359 1 1 1 1 1 1 F F F
4292024-01-24T19:05:34.935Z 18 360-379 1 1 1 1 1 1 F F F
4302024-01-24T19:05:34.935Z 19 380-399 1 1 1 1 1 1 F F F
4312024-01-24T19:05:34.936Z 20 400-419 2 2 2 2 2 2 F F F
4322024-01-24T19:05:34.936Z 21 420-439 2 2 2 2 2 2 F F F
4332024-01-24T19:05:34.936Z 22 440-459 1 1 1 1 1 1 F F F
4342024-01-24T19:05:34.936Z 23 460-479 1 1 1 1 1 1 F F F
4352024-01-24T19:05:34.937Z 24 480-499 3 3 3 3 3 3 F F F
4362024-01-24T19:05:34.937Z 25 500-519 3 3 3 3 3 3 F F F
4372024-01-24T19:05:34.937Z 26 520-539 2 2 2 2 2 2 F F F
4382024-01-24T19:05:34.937Z 27 540-559 1 1 1 1 1 1 F F F
4392024-01-24T19:05:34.937Z 28 560-579 1 1 1 1 1 1 F F F
4402024-01-24T19:05:34.938Z 29 580-599 1 1 1 1 1 1 F F F
4412024-01-24T19:05:34.938ZMax gen: 3, Max flush: 3
4422024-01-24T19:05:34.946Z
4432024-01-24T19:05:34.947ZBegin loop 2
4442024-01-24T19:05:34.947ZDownstairs to restart: 2
4452024-01-24T19:05:34.960Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.603304268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145}
4462024-01-24T19:05:34.960Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.603720709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145}
4472024-01-24T19:05:34.960Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.603749288Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145}
4482024-01-24T19:05:34.961Z{"msg":"Crucible fafebc01-0846-43dc-ac24-c29ba56bc539 has session id: 532c0b72-ec4d-4201-8641-09c4a2c08568","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.603793753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4492024-01-24T19:05:34.961Z{"msg":"Upstairs opts: Upstairs UUID: fafebc01-0846-43dc-ac24-c29ba56bc539, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.60383168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4502024-01-24T19:05:34.961Z{"msg":"Crucible stats registered with UUID: fafebc01-0846-43dc-ac24-c29ba56bc539","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.603881559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4512024-01-24T19:05:34.961ZCrucible runtime is spawned
4522024-01-24T19:05:34.961Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.60431955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145}
4532024-01-24T19:05:34.981Z{"msg":"fafebc01-0846-43dc-ac24-c29ba56bc539 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.604433077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4542024-01-24T19:05:34.981Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.60451372Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4552024-01-24T19:05:34.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.604656609Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4562024-01-24T19:05:34.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.604750125Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4572024-01-24T19:05:34.984Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.604797898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4582024-01-24T19:05:34.984Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.60499456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4592024-01-24T19:05:34.984Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605075286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4602024-01-24T19:05:34.984Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605119746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4612024-01-24T19:05:34.984Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605163425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4622024-01-24T19:05:34.985Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605205455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4632024-01-24T19:05:34.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605233555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4642024-01-24T19:05:34.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605280222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4652024-01-24T19:05:34.986Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.605306788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4662024-01-24T19:05:34.986Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606339648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4672024-01-24T19:05:34.986Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606483489Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4682024-01-24T19:05:34.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606600063Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4692024-01-24T19:05:34.987Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606645342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4702024-01-24T19:05:34.987Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606815755Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4712024-01-24T19:05:34.987Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.606949774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4722024-01-24T19:05:34.987Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.607710937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4732024-01-24T19:05:34.987Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.607791457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4742024-01-24T19:05:34.988Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.607927271Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4752024-01-24T19:05:34.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608611051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4762024-01-24T19:05:34.988Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608674278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4772024-01-24T19:05:34.988Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608748906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4782024-01-24T19:05:34.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608808502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4792024-01-24T19:05:34.989Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608839915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4802024-01-24T19:05:34.989Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608911649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4812024-01-24T19:05:35.018Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.608953468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4822024-01-24T19:05:35.019Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609014025Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4832024-01-24T19:05:35.019Z{"msg":"[0]R flush_numbers[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609042631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4842024-01-24T19:05:35.019Z{"msg":"[0]R generation[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609065836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4852024-01-24T19:05:35.019Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609111684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4862024-01-24T19:05:35.019Z{"msg":"[1]R flush_numbers[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609138935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4872024-01-24T19:05:35.020Z{"msg":"[1]R generation[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609186372Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4882024-01-24T19:05:35.020Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4892024-01-24T19:05:35.020Z,"time":"2024-01-24T19:05:33.60921368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4902024-01-24T19:05:35.021Z{"msg":"[2]R flush_numbers[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609260167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4912024-01-24T19:05:35.021Z{"msg":"[2]R generation[0..12]: [1, 3, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609286835Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4922024-01-24T19:05:35.021Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609310622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4932024-01-24T19:05:35.022Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609353386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4942024-01-24T19:05:35.022Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609379337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4952024-01-24T19:05:35.022Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609401017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4962024-01-24T19:05:35.022Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609444614Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4972024-01-24T19:05:35.022Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.60947046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4982024-01-24T19:05:35.022Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609512072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
4992024-01-24T19:05:35.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609537152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5002024-01-24T19:05:35.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609586356Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5012024-01-24T19:05:35.024Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609614215Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5022024-01-24T19:05:35.024Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609634817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5032024-01-24T19:05:35.024Z{"msg":"fafebc01-0846-43dc-ac24-c29ba56bc539 is now active with session: 532c0b72-ec4d-4201-8641-09c4a2c08568","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609656062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5042024-01-24T19:05:35.024Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609701283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145,"session_id":"532c0b72-ec4d-4201-8641-09c4a2c08568"}
5052024-01-24T19:05:35.024Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:33.609724759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1145}
5062024-01-24T19:05:35.025ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5072024-01-24T19:05:35.025ZLoading write count information from file "/tmp/test_repair_verify.data"
5082024-01-24T19:05:35.025ZRun Repair workload
5092024-01-24T19:05:35.025Z01/30 Write block 305 len 8192 data: 2 2
5102024-01-24T19:05:35.025Z02/30 Flush
5112024-01-24T19:05:35.025Z03/30 Flush
5122024-01-24T19:05:35.026Z04/30 Write block 289 len 28672 data: 2 2 2 2 2 2 2
5132024-01-24T19:05:35.026Z05/30 Read block 303 len 28672
5142024-01-24T19:05:35.026Z06/30 Read block 320 len 28672
5152024-01-24T19:05:35.026Z07/30 Write block 129 len 8192 data: 2 2
5162024-01-24T19:05:35.027Z08/30 Write block 151 len 12288 data: 2 2 2
5172024-01-24T19:05:35.027Z09/30 Read block 136 len 16384
5182024-01-24T19:05:35.027Z10/30 Read block 368 len 40960
5192024-01-24T19:05:35.027Z11/30 Read block 521 len 28672
5202024-01-24T19:05:35.027Z12/30 Write block 225 len 36864 data: 3 3 3 2 2 2 2 2 2
5212024-01-24T19:05:35.027Z13/30 Write block 387 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5222024-01-24T19:05:35.027Z14/30 Write block 336 len 40960 data: 3 3 2 2 2 2 2 2 2 2
5232024-01-24T19:05:35.027Z15/30 Read block 335 len 32768
5242024-01-24T19:05:35.028Z16/30 Flush
5252024-01-24T19:05:35.028Z17/30 Write block 44 len 8192 data: 2 2
5262024-01-24T19:05:35.028Z18/30 Read block 186 len 12288
5272024-01-24T19:05:35.028Z19/30 Write block 533 len 28672 data: 2 2 2 2 2 2 2
5282024-01-24T19:05:35.028Z20/30 Read block 110 len 4096
5292024-01-24T19:05:35.028Z21/30 Write block 546 len 4096 data: 2
5302024-01-24T19:05:35.028Z22/30 Read block 194 len 20480
5312024-01-24T19:05:35.028Z23/30 Read block 297 len 40960
5322024-01-24T19:05:35.462Z24/30 Write block 524 len 28672 data: 3 3 3 2 2 2 2
5332024-01-24T19:05:35.462Z25/30 Read block 583 len 24576
5342024-01-24T19:05:35.462Z26/30 Write block 281 len 16384 data: 2 2 2 2
5352024-01-24T19:05:35.462Z27/30 Read block 4 len 16384
5362024-01-24T19:05:35.462Z28/30 Read block 341 len 32768
5372024-01-24T19:05:35.462Z29/30 Read block 197 len 4096
5382024-01-24T19:05:35.477Z30/30 Write block 406 len 24576 data: 2 2 2 2 2 2
5392024-01-24T19:05:35.477Z----------------------------------------------------------------
5402024-01-24T19:05:35.477Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:28
5412024-01-24T19:05:35.477ZStates: Active Active Active
5422024-01-24T19:05:35.477ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5432024-01-24T19:05:35.478Z 3 Acked 1002 Flush 0 Done Done Sent false
5442024-01-24T19:05:35.478Z 4 Acked 1003 Write 7 Done Done Sent false
5452024-01-24T19:05:35.478Z 5 Acked 1004 Read 7 Done Done Sent false
5462024-01-24T19:05:35.478Z 6 Acked 1005 Read 7 Done Done Sent false
5472024-01-24T19:05:35.478Z 7 Acked 1006 Write 2 Done Done Sent false
5482024-01-24T19:05:35.478Z 8 Acked 1007 Write 3 Done Done Sent false
5492024-01-24T19:05:35.478Z 9 Acked 1008 Read 4 Done Done Sent false
5502024-01-24T19:05:35.478Z 10 Acked 1009 Read 10 Done Done Sent false
5512024-01-24T19:05:35.478Z 11 Acked 1010 Read 7 Done Done Sent false
5522024-01-24T19:05:35.480Z 12 Acked 1011 Write 9 Done Done Sent false
5532024-01-24T19:05:35.480Z 13 Acked 1012 Write 10 Done Done Sent false
5542024-01-24T19:05:35.480Z 14 Acked 1013 Write 10 Done Done Sent false
5552024-01-24T19:05:35.480Z 15 Acked 1014 Read 8 Done Done Sent false
5562024-01-24T19:05:35.480Z 16 Acked 1015 Flush 0 Done Done Sent false
5572024-01-24T19:05:35.480Z 17 Acked 1016 Write 2 Done Done Sent false
5582024-01-24T19:05:35.481Z 18 Acked 1017 Read 3 Done Done Sent false
5592024-01-24T19:05:35.481Z 19 Acked 1018 Write 7 Done Done Sent false
5602024-01-24T19:05:35.481Z 20 Acked 1019 Read 1 Done Done Sent false
5612024-01-24T19:05:35.481Z 21 Acked 1020 Write 1 Done Done Sent false
5622024-01-24T19:05:35.481Z 22 Acked 1021 Read 5 Done Done Sent false
5632024-01-24T19:05:35.481Z 23 Acked 1022 Read 10 Done Done Sent false
5642024-01-24T19:05:35.481Z 24 Acked 1023 Write 7 Done Done Sent false
5652024-01-24T19:05:35.481Z 25 Acked 1024 Read 6 Done Done Sent false
5662024-01-24T19:05:35.482Z 26 Acked 1025 Write 4 Done Done Sent false
5672024-01-24T19:05:35.482Z 27 Acked 1026 Read 4 Done Done Sent false
5682024-01-24T19:05:35.482Z 28 Acked 1027 Read 8 Done Done Sent false
5692024-01-24T19:05:35.482Z 29 Acked 1028 Read 1 Done Done Sent false
5702024-01-24T19:05:35.482Z 30 Acked 1029 Write 6 Sent Sent Sent false
5712024-01-24T19:05:35.482Z STATES DS:0 DS:1 DS:2 TOTAL
5722024-01-24T19:05:35.482Z New 0 0 0 0
5732024-01-24T19:05:35.482Z Sent 1 1 28 30
5742024-01-24T19:05:35.482Z Done 27 27 0 54
5752024-01-24T19:05:35.482Z Skipped 0 0 0 0
5762024-01-24T19:05:35.483Z Error 0 0 0 0
5772024-01-24T19:05:35.483ZLast Flush: 1015 1015 1001
5782024-01-24T19:05:35.483ZDownstairs last five completed: 1001 1000
5792024-01-24T19:05:35.483ZUpstairs last five completed: 30 29 28 27 26
5802024-01-24T19:05:35.483ZWrote out file "/tmp/test_repair_verify.data"
5812024-01-24T19:05:35.483Z
5822024-01-24T19:05:35.483ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5832024-01-24T19:05:35.925ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5842024-01-24T19:05:35.925Z 0 000-019 1 1 1 1 1 1 F F F
5852024-01-24T19:05:35.925Z 1 020-039 3 3 3 3 3 3 F F F
5862024-01-24T19:05:35.925Z 2 040-059 1 1 1 1 1 1 T T F <---
5872024-01-24T19:05:35.926Z 3 060-079 1 1 1 1 1 1 F F F
5882024-01-24T19:05:35.926Z 4 080-099 2 2 2 2 2 2 F F F
5892024-01-24T19:05:35.926Z 5 100-119 2 2 2 2 2 2 F F F
5902024-01-24T19:05:35.926Z 6 120-139 4 4 1 6 6 1 F F F <---
5912024-01-24T19:05:35.926Z 7 140-159 4 4 2 6 6 2 F F F <---
5922024-01-24T19:05:35.926Z 8 160-179 1 1 1 1 1 1 F F F
5932024-01-24T19:05:35.927Z 9 180-199 1 1 1 1 1 1 F F F
5942024-01-24T19:05:35.927Z 10 200-219 2 2 2 2 2 2 F F F
5952024-01-24T19:05:35.927Z 11 220-239 4 4 2 6 6 2 F F F <---
5962024-01-24T19:05:35.927Z 12 240-259 2 2 2 2 2 2 F F F
5972024-01-24T19:05:35.927Z 13 260-279 1 1 1 1 1 1 F F F
5982024-01-24T19:05:35.927Z 14 280-299 4 4 1 6 6 1 T T F <---
5992024-01-24T19:05:35.927Z 15 300-319 4 4 4 4 4 4 F F F
6002024-01-24T19:05:35.927Z 16 320-339 4 4 3 6 6 3 F F F <---
6012024-01-24T19:05:35.927Z 17 340-359 4 4 1 6 6 1 F F F <---
6022024-01-24T19:05:35.928Z 18 360-379 1 1 1 1 1 1 F F F
6032024-01-24T19:05:35.928Z 19 380-399 4 4 1 6 6 1 F F F <---
6042024-01-24T19:05:35.928Z 20 400-419 2 2 2 2 2 2 T T F <---
6052024-01-24T19:05:35.928Z 21 420-439 2 2 2 2 2 2 F F F
6062024-01-24T19:05:35.928Z 22 440-459 1 1 1 1 1 1 F F F
6072024-01-24T19:05:35.928Z 23 460-479 1 1 1 1 1 1 F F F
6082024-01-24T19:05:35.928Z 24 480-499 3 3 3 3 3 3 F F F
6092024-01-24T19:05:35.928Z 25 500-519 3 3 3 3 3 3 F F F
6102024-01-24T19:05:35.928Z 26 520-539 2 2 2 2 2 2 T T F <---
6112024-01-24T19:05:35.928Z 27 540-559 1 1 1 1 1 1 T T F <---
6122024-01-24T19:05:35.928Z 28 560-579 1 1 1 1 1 1 F F F
6132024-01-24T19:05:35.928Z 29 580-599 1 1 1 1 1 1 F F F
6142024-01-24T19:05:35.928ZMax gen: 4, Max flush: 6
6152024-01-24T19:05:35.929ZError: Difference in extent metadata found!
6162024-01-24T19:05:35.929ZA Difference in extent metadata is expected here
6172024-01-24T19:05:35.929ZOn loop 2
6182024-01-24T19:05:35.929Z
6192024-01-24T19:05:35.929ZVerifying data now
6202024-01-24T19:05:40.072ZNo change after verify
6212024-01-24T19:05:40.072ZLoop: 2 Downstairs dump after verify (and repair):
6222024-01-24T19:05:40.102ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6232024-01-24T19:05:40.103Z 0 000-019 1 1 1 1 1 1 F F F
6242024-01-24T19:05:40.128Z 1 020-039 3 3 3 3 3 3 F F F
6252024-01-24T19:05:40.129Z 2 040-059 5 5 5 7 7 7 F F F
6262024-01-24T19:05:40.129Z 3 060-079 1 1 1 1 1 1 F F F
6272024-01-24T19:05:40.129Z 4 080-099 2 2 2 2 2 2 F F F
6282024-01-24T19:05:40.129Z 5 100-119 2 2 2 2 2 2 F F F
6292024-01-24T19:05:40.129Z 6 120-139 4 4 4 6 6 6 F F F
6302024-01-24T19:05:40.129Z 7 140-159 4 4 4 6 6 6 F F F
6312024-01-24T19:05:40.129Z 8 160-179 1 1 1 1 1 1 F F F
6322024-01-24T19:05:40.129Z 9 180-199 1 1 1 1 1 1 F F F
6332024-01-24T19:05:40.129Z 10 200-219 2 2 2 2 2 2 F F F
6342024-01-24T19:05:40.129Z 11 220-239 4 4 4 6 6 6 F F F
6352024-01-24T19:05:40.129Z 12 240-259 2 2 2 2 2 2 F F F
6362024-01-24T19:05:40.129Z 13 260-279 1 1 1 1 1 1 F F F
6372024-01-24T19:05:40.129Z 14 280-299 5 5 5 7 7 7 F F F
6382024-01-24T19:05:40.129Z 15 300-319 4 4 4 4 4 4 F F F
6392024-01-24T19:05:40.129Z 16 320-339 4 4 4 6 6 6 F F F
6402024-01-24T19:05:40.129Z 17 340-359 4 4 4 6 6 6 F F F
6412024-01-24T19:05:40.130Z 18 360-379 1 1 1 1 1 1 F F F
6422024-01-24T19:05:40.130Z 19 380-399 4 4 4 6 6 6 F F F
6432024-01-24T19:05:40.130Z 20 400-419 5 5 5 7 7 7 F F F
6442024-01-24T19:05:40.130Z 21 420-439 2 2 2 2 2 2 F F F
6452024-01-24T19:05:40.130Z 22 440-459 1 1 1 1 1 1 F F F
6462024-01-24T19:05:40.130Z 23 460-479 1 1 1 1 1 1 F F F
6472024-01-24T19:05:40.130Z 24 480-499 3 3 3 3 3 3 F F F
6482024-01-24T19:05:40.130Z 25 500-519 3 3 3 3 3 3 F F F
6492024-01-24T19:05:40.130Z 26 520-539 5 5 5 7 7 7 F F F
6502024-01-24T19:05:40.130Z 27 540-559 5 5 5 7 7 7 F F F
6512024-01-24T19:05:40.130Z 28 560-579 1 1 1 1 1 1 F F F
6522024-01-24T19:05:40.130Z 29 580-599 1 1 1 1 1 1 F F F
6532024-01-24T19:05:40.130ZMax gen: 5, Max flush: 7
6542024-01-24T19:05:40.130Z
6552024-01-24T19:05:40.131ZBegin loop 3
6562024-01-24T19:05:40.131ZDownstairs to restart: 2
6572024-01-24T19:05:40.143Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782030515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153}
6582024-01-24T19:05:40.143Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782491325Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153}
6592024-01-24T19:05:40.143Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782544954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153}
6602024-01-24T19:05:40.143Z{"msg":"Crucible 3918807a-aae6-4884-bc97-08b85217a649 has session id: 742b0451-21fb-43d7-a24b-1be389acdb1b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782570088Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6612024-01-24T19:05:40.143Z{"msg":"Upstairs opts: Upstairs UUID: 3918807a-aae6-4884-bc97-08b85217a649, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782605236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6622024-01-24T19:05:40.144Z{"msg":"Crucible stats registered with UUID: 3918807a-aae6-4884-bc97-08b85217a649","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782666757Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6632024-01-24T19:05:40.144ZCrucible runtime is spawned
6642024-01-24T19:05:40.144Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.782971376Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153}
6652024-01-24T19:05:40.144Z{"msg":"3918807a-aae6-4884-bc97-08b85217a649 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783136203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6662024-01-24T19:05:40.144Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783192123Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6672024-01-24T19:05:40.144Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783234509Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6682024-01-24T19:05:40.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783315627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6692024-01-24T19:05:40.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783359814Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6702024-01-24T19:05:40.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783428565Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6712024-01-24T19:05:40.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783474699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6722024-01-24T19:05:40.146Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783539333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6732024-01-24T19:05:40.146Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783680292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6742024-01-24T19:05:40.147Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783743541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6752024-01-24T19:05:40.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783799186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6762024-01-24T19:05:40.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783843574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6772024-01-24T19:05:40.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.783933515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6782024-01-24T19:05:40.148Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.784983718Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6792024-01-24T19:05:40.148Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.785047435Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6802024-01-24T19:05:40.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.785248551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6812024-01-24T19:05:40.149Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.785360221Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6822024-01-24T19:05:40.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.785437311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6832024-01-24T19:05:40.149Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.785468201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6842024-01-24T19:05:40.149Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.786215632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6852024-01-24T19:05:40.149Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.786361348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6862024-01-24T19:05:40.149Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.78648339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6872024-01-24T19:05:40.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787229573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6882024-01-24T19:05:40.151Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787287801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6892024-01-24T19:05:40.151Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787396432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6902024-01-24T19:05:40.151Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787461978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6912024-01-24T19:05:40.151Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787489713Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6922024-01-24T19:05:40.151Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787534283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6932024-01-24T19:05:40.151Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787558063Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6942024-01-24T19:05:40.151Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787601364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6952024-01-24T19:05:40.152Z{"msg":"[0]R flush_numbers[0..12]: [1, 3, 7, 1, 2, 2, 6, 6, 1, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787627107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6962024-01-24T19:05:40.152Z{"msg":"[0]R generation[0..12]: [1, 3, 5, 1, 2, 2, 4, 4, 1, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.78765123Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6972024-01-24T19:05:40.152Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787675004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6982024-01-24T19:05:40.152Z{"msg":"[1]R flush_numbers[0..12]: [1, 3, 7, 1, 2, 2, 6, 6, 1, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787719194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
6992024-01-24T19:05:40.152Z{"msg":"[1]R generation[0..12]: [1, 3, 5, 1, 2, 2, 4, 4, 1, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.78774492Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7002024-01-24T19:05:40.152Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
7012024-01-24T19:05:40.152Z,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787812232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7022024-01-24T19:05:40.152Z{"msg":"[2]R flush_numbers[0..12]: [1, 3, 7, 1, 2, 2, 6, 6, 1, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787895388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7032024-01-24T19:05:40.153Z{"msg":"[2]R generation[0..12]: [1, 3, 5, 1, 2, 2, 4, 4, 1, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.787958728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7042024-01-24T19:05:40.153Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788034127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7052024-01-24T19:05:40.153Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788105623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7062024-01-24T19:05:40.153Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788134246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7072024-01-24T19:05:40.153Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788178426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7082024-01-24T19:05:40.154Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788204211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7092024-01-24T19:05:40.155Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788263255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7102024-01-24T19:05:40.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788331106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7112024-01-24T19:05:40.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788363252Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7122024-01-24T19:05:40.156Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.78840799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7132024-01-24T19:05:40.156Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788447468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7142024-01-24T19:05:40.156Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.78851424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7152024-01-24T19:05:40.156Z{"msg":"3918807a-aae6-4884-bc97-08b85217a649 is now active with session: 742b0451-21fb-43d7-a24b-1be389acdb1b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788549674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7162024-01-24T19:05:40.156Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7172024-01-24T19:05:40.156Z,"time":"2024-01-24T19:05:38.788636749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153,"session_id":"742b0451-21fb-43d7-a24b-1be389acdb1b"}
7182024-01-24T19:05:40.156Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:38.788709366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1153}
7192024-01-24T19:05:40.157ZLoading write count information from file "/tmp/test_repair_verify.data"
7202024-01-24T19:05:40.158ZRun Repair workload
7212024-01-24T19:05:40.158Z01/30 Write block 41 len 24576 data: 2 2 2 3 3 2
7222024-01-24T19:05:40.158Z02/30 Write block 246 len 8192 data: 3 3
7232024-01-24T19:05:40.165Z03/30 Read block 492 len 36864
7242024-01-24T19:05:40.166Z04/30 Read block 426 len 24576
7252024-01-24T19:05:40.166Z05/30 Read block 67 len 32768
7262024-01-24T19:05:40.166Z06/30 Read block 507 len 32768
7272024-01-24T19:05:40.166Z07/30 Write block 54 len 28672 data: 2 2 2 2 2 2 2
7282024-01-24T19:05:40.166Z08/30 Read block 357 len 28672
7292024-01-24T19:05:40.166Z09/30 Write block 356 len 4096 data: 2
7302024-01-24T19:05:40.166Z10/30 Write block 393 len 40960 data: 3 3 3 3 2 2 2 2 2 2
7312024-01-24T19:05:40.167Z11/30 Read block 351 len 20480
7322024-01-24T19:05:40.167Z12/30 Read block 466 len 36864
7332024-01-24T19:05:40.167Z13/30 Read block 498 len 12288
7342024-01-24T19:05:40.167Z14/30 Flush
7352024-01-24T19:05:40.167Z15/30 Write block 168 len 32768 data: 2 2 2 2 2 2 2 2
7362024-01-24T19:05:40.167Z16/30 Write block 8 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7372024-01-24T19:05:40.168Z17/30 Write block 386 len 16384 data: 2 3 3 3
7382024-01-24T19:05:40.175Z18/30 Read block 101 len 20480
7392024-01-24T19:05:40.175Z19/30 Read block 485 len 36864
7402024-01-24T19:05:40.175Z20/30 Read block 361 len 24576
7412024-01-24T19:05:40.175Z21/30 Read block 491 len 20480
7422024-01-24T19:05:40.176Z22/30 Read block 574 len 8192
7432024-01-24T19:05:40.185Z23/30 Read block 320 len 16384
7442024-01-24T19:05:40.185Z24/30 Read block 442 len 28672
7452024-01-24T19:05:40.185Z25/30 Read block 303 len 24576
7462024-01-24T19:05:40.185Z26/30 Write block 82 len 24576 data: 2 2 2 2 2 2
7472024-01-24T19:05:40.185Z27/30 Write block 458 len 24576 data: 2 2 2 2 2 2
7482024-01-24T19:05:40.185Z28/30 Write block 97 len 16384 data: 3 3 3 3
7492024-01-24T19:05:40.186Z29/30 Write block 541 len 20480 data: 2 2 2 2 2
7502024-01-24T19:05:40.186Z30/30 Read block 270 len 16384
7512024-01-24T19:05:40.201Z----------------------------------------------------------------
7522024-01-24T19:05:40.202Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7532024-01-24T19:05:40.202ZStates: Active Active Active
7542024-01-24T19:05:40.202ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7552024-01-24T19:05:40.202Z 1 Acked 1000 Write 6 Done Done Done false
7562024-01-24T19:05:40.202Z 2 Acked 1001 Write 2 Done Done Done false
7572024-01-24T19:05:40.203Z 3 Acked 1002 Read 9 Done Done Done false
7582024-01-24T19:05:40.203Z 4 Acked 1003 Read 6 Done Done Sent false
7592024-01-24T19:05:40.203Z 5 Acked 1004 Read 8 Done Done Sent false
7602024-01-24T19:05:40.203Z 6 Acked 1005 Read 8 Done Done Sent false
7612024-01-24T19:05:40.204Z 7 Acked 1006 Write 7 Done Done Sent false
7622024-01-24T19:05:40.204Z 8 Acked 1007 Read 7 Done Done Sent false
7632024-01-24T19:05:40.204Z 9 Acked 1008 Write 1 Done Done Sent false
7642024-01-24T19:05:40.204Z 10 Acked 1009 Write 10 Done Done Sent false
7652024-01-24T19:05:40.204Z 11 Acked 1010 Read 5 Done Done Sent false
7662024-01-24T19:05:40.205Z 12 Acked 1011 Read 9 Done Done Sent false
7672024-01-24T19:05:40.205Z 13 Acked 1012 Read 3 Done Done Sent false
7682024-01-24T19:05:40.206Z 14 Acked 1013 Flush 0 Done Done Sent false
7692024-01-24T19:05:40.207Z 15 Acked 1014 Write 8 Done Done Sent false
7702024-01-24T19:05:40.207Z 16 Acked 1015 Write 10 Done Done Sent false
7712024-01-24T19:05:40.207Z 17 Acked 1016 Write 4 Done Done Sent false
7722024-01-24T19:05:40.207Z 18 Acked 1017 Read 5 Done Done Sent false
7732024-01-24T19:05:40.207Z 19 Acked 1018 Read 9 Done Done Sent false
7742024-01-24T19:05:40.207Z 20 Acked 1019 Read 6 Done Done Sent false
7752024-01-24T19:05:40.207Z 21 Acked 1020 Read 5 Done Done Sent false
7762024-01-24T19:05:40.207Z 22 Acked 1021 Read 2 Done Done Sent false
7772024-01-24T19:05:40.207Z 23 Acked 1022 Read 4 Done Done Sent false
7782024-01-24T19:05:40.207Z 24 Acked 1023 Read 7 Done Done Sent false
7792024-01-24T19:05:40.208Z 25 Acked 1024 Read 6 Done Done Sent false
7802024-01-24T19:05:40.208Z 26 Acked 1025 Write 6 Done Done Sent false
7812024-01-24T19:05:40.208Z 27 Acked 1026 Write 6 Done Done Sent false
7822024-01-24T19:05:40.208Z 28 Acked 1027 Write 4 Done Done Sent false
7832024-01-24T19:05:40.208Z 29 Acked 1028 Write 5 Done Done Sent false
7842024-01-24T19:05:40.208Z 30 Acked 1029 Read 4 Done Done Sent false
7852024-01-24T19:05:40.208Z STATES DS:0 DS:1 DS:2 TOTAL
7862024-01-24T19:05:40.208Z New 0 0 0 0
7872024-01-24T19:05:40.208Z Sent 0 0 27 27
7882024-01-24T19:05:40.208Z Done 30 30 3 63
7892024-01-24T19:05:40.208Z Skipped 0 0 0 0
7902024-01-24T19:05:40.208Z Error 0 0 0 0
7912024-01-24T19:05:40.209ZLast Flush: 1013 1013 0
7922024-01-24T19:05:40.209ZDownstairs last five completed:
7932024-01-24T19:05:40.209ZUpstairs last five completed: 30 29 28 27 26
7942024-01-24T19:05:40.209ZWrote out file "/tmp/test_repair_verify.data"
7952024-01-24T19:05:40.209Z
7962024-01-24T19:05:40.227ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
7972024-01-24T19:05:40.241ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
7982024-01-24T19:05:40.241Z 0 000-019 1 1 1 1 1 1 T T F <---
7992024-01-24T19:05:40.241Z 1 020-039 3 3 3 3 3 3 F F F
8002024-01-24T19:05:40.241Z 2 040-059 6 6 5 8 8 7 F F T <---
8012024-01-24T19:05:40.241Z 3 060-079 6 6 1 8 8 1 F F F <---
8022024-01-24T19:05:40.241Z 4 080-099 2 2 2 2 2 2 T T F <---
8032024-01-24T19:05:40.241Z 5 100-119 2 2 2 2 2 2 T T F <---
8042024-01-24T19:05:40.241Z 6 120-139 4 4 4 6 6 6 F F F
8052024-01-24T19:05:40.242Z 7 140-159 4 4 4 6 6 6 F F F
8062024-01-24T19:05:40.242Z 8 160-179 1 1 1 1 1 1 T T F <---
8072024-01-24T19:05:40.242Z 9 180-199 1 1 1 1 1 1 F F F
8082024-01-24T19:05:40.242Z 10 200-219 2 2 2 2 2 2 F F F
8092024-01-24T19:05:40.242Z 11 220-239 4 4 4 6 6 6 F F F
8102024-01-24T19:05:40.243Z 12 240-259 6 6 2 8 8 2 F F T <---
8112024-01-24T19:05:40.243Z 13 260-279 1 1 1 1 1 1 F F F
8122024-01-24T19:05:40.243Z 14 280-299 5 5 5 7 7 7 F F F
8132024-01-24T19:05:40.243Z 15 300-319 4 4 4 4 4 4 F F F
8142024-01-24T19:05:40.243Z 16 320-339 4 4 4 6 6 6 F F F
8152024-01-24T19:05:40.243Z 17 340-359 6 6 4 8 8 6 F F F <---
8162024-01-24T19:05:40.243Z 18 360-379 1 1 1 1 1 1 F F F
8172024-01-24T19:05:40.243Z 19 380-399 6 6 4 8 8 6 T T F <---
8182024-01-24T19:05:40.244Z 20 400-419 6 6 5 8 8 7 F F F <---
8192024-01-24T19:05:40.244Z 21 420-439 2 2 2 2 2 2 F F F
8202024-01-24T19:05:40.244Z 22 440-459 1 1 1 1 1 1 T T F <---
8212024-01-24T19:05:40.244Z 23 460-479 1 1 1 1 1 1 T T F <---
8222024-01-24T19:05:40.244Z 24 480-499 3 3 3 3 3 3 F F F
8232024-01-24T19:05:40.244Z 25 500-519 3 3 3 3 3 3 F F F
8242024-01-24T19:05:40.244Z 26 520-539 5 5 5 7 7 7 F F F
8252024-01-24T19:05:40.244Z 27 540-559 5 5 5 7 7 7 T T F <---
8262024-01-24T19:05:40.244Z 28 560-579 1 1 1 1 1 1 F F F
8272024-01-24T19:05:40.245Z 29 580-599 1 1 1 1 1 1 F F F
8282024-01-24T19:05:40.245ZMax gen: 6, Max flush: 8
8292024-01-24T19:05:40.245ZError: Difference in extent metadata found!
8302024-01-24T19:05:40.245ZA Difference in extent metadata is expected here
8312024-01-24T19:05:40.245ZOn loop 3
8322024-01-24T19:05:40.245Z
8332024-01-24T19:05:40.245ZVerifying data now
8342024-01-24T19:05:46.392ZNo change after verify
8352024-01-24T19:05:46.393ZLoop: 3 Downstairs dump after verify (and repair):
8362024-01-24T19:05:46.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8372024-01-24T19:05:46.824Z 0 000-019 7 7 7 9 9 9 F F F
8382024-01-24T19:05:46.833Z 1 020-039 3 3 3 3 3 3 F F F
8392024-01-24T19:05:46.833Z 2 040-059 6 6 6 8 8 8 F F F
8402024-01-24T19:05:46.834Z 3 060-079 6 6 6 8 8 8 F F F
8412024-01-24T19:05:46.834Z 4 080-099 7 7 7 9 9 9 F F F
8422024-01-24T19:05:46.834Z 5 100-119 7 7 7 9 9 9 F F F
8432024-01-24T19:05:46.836Z 6 120-139 4 4 4 6 6 6 F F F
8442024-01-24T19:05:46.836Z 7 140-159 4 4 4 6 6 6 F F F
8452024-01-24T19:05:46.836Z 8 160-179 7 7 7 9 9 9 F F F
8462024-01-24T19:05:46.836Z 9 180-199 1 1 1 1 1 1 F F F
8472024-01-24T19:05:46.836Z 10 200-219 2 2 2 2 2 2 F F F
8482024-01-24T19:05:46.836Z 11 220-239 4 4 4 6 6 6 F F F
8492024-01-24T19:05:46.836Z 12 240-259 6 6 6 8 8 8 F F F
8502024-01-24T19:05:46.836Z 13 260-279 1 1 1 1 1 1 F F F
8512024-01-24T19:05:46.836Z 14 280-299 5 5 5 7 7 7 F F F
8522024-01-24T19:05:46.836Z 15 300-319 4 4 4 4 4 4 F F F
8532024-01-24T19:05:46.836Z 16 320-339 4 4 4 6 6 6 F F F
8542024-01-24T19:05:46.836Z 17 340-359 6 6 6 8 8 8 F F F
8552024-01-24T19:05:46.836Z 18 360-379 1 1 1 1 1 1 F F F
8562024-01-24T19:05:46.836Z 19 380-399 7 7 7 9 9 9 F F F
8572024-01-24T19:05:46.836Z 20 400-419 6 6 6 8 8 8 F F F
8582024-01-24T19:05:46.837Z 21 420-439 2 2 2 2 2 2 F F F
8592024-01-24T19:05:46.837Z 22 440-459 7 7 7 9 9 9 F F F
8602024-01-24T19:05:46.837Z 23 460-479 7 7 7 9 9 9 F F F
8612024-01-24T19:05:46.837Z 24 480-499 3 3 3 3 3 3 F F F
8622024-01-24T19:05:46.837Z 25 500-519 3 3 3 3 3 3 F F F
8632024-01-24T19:05:46.837Z 26 520-539 5 5 5 7 7 7 F F F
8642024-01-24T19:05:46.837Z 27 540-559 7 7 7 9 9 9 F F F
8652024-01-24T19:05:46.838Z 28 560-579 1 1 1 1 1 1 F F F
8662024-01-24T19:05:46.838Z 29 580-599 1 1 1 1 1 1 F F F
8672024-01-24T19:05:46.838ZMax gen: 7, Max flush: 9
8682024-01-24T19:05:46.838Z
8692024-01-24T19:05:46.838ZBegin loop 4
8702024-01-24T19:05:46.838ZDownstairs to restart: 1
8712024-01-24T19:05:46.838Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104132605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161}
8722024-01-24T19:05:46.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104653151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161}
8732024-01-24T19:05:46.838Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104683069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161}
8742024-01-24T19:05:46.839Z{"msg":"Crucible 822b36b3-f723-4cf6-b428-a5220af6ab97 has session id: acdf9ab9-a9e4-43cf-9756-ca8246b0563a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104704256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8752024-01-24T19:05:46.839Z{"msg":"Upstairs opts: Upstairs UUID: 822b36b3-f723-4cf6-b428-a5220af6ab97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104743944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8762024-01-24T19:05:46.839Z{"msg":"Crucible stats registered with UUID: 822b36b3-f723-4cf6-b428-a5220af6ab97","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.104810785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8772024-01-24T19:05:46.839ZCrucible runtime is spawned
8782024-01-24T19:05:46.839Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105039097Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161}
8792024-01-24T19:05:46.839Z{"msg":"822b36b3-f723-4cf6-b428-a5220af6ab97 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105319679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8802024-01-24T19:05:46.839Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105380169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8812024-01-24T19:05:46.839Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105409095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8822024-01-24T19:05:46.839Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105433479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8832024-01-24T19:05:46.839Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.10550194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8842024-01-24T19:05:46.839Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105528438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8852024-01-24T19:05:46.839Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.10555528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8862024-01-24T19:05:46.839Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105651373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8872024-01-24T19:05:46.839Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105697631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8882024-01-24T19:05:46.839Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105741728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8892024-01-24T19:05:46.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105783151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8902024-01-24T19:05:46.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.105943864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8912024-01-24T19:05:46.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.106059784Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8922024-01-24T19:05:46.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.107059413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8932024-01-24T19:05:46.840Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.107116149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8942024-01-24T19:05:46.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.10721624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8952024-01-24T19:05:46.840Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.107329977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8962024-01-24T19:05:46.840Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.107905876Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8972024-01-24T19:05:46.840Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.108016476Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8982024-01-24T19:05:46.841Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.108227676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
8992024-01-24T19:05:46.842Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.108350141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9002024-01-24T19:05:46.842Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109091061Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9012024-01-24T19:05:46.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109411943Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9022024-01-24T19:05:46.842Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109475083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9032024-01-24T19:05:46.842Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.1095444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9042024-01-24T19:05:46.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109581779Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9052024-01-24T19:05:46.842Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109606715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9062024-01-24T19:05:46.842Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109655365Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9072024-01-24T19:05:46.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.109943324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9082024-01-24T19:05:46.842Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110037754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9092024-01-24T19:05:46.842Z{"msg":"[0]R flush_numbers[0..12]: [9, 3, 8, 8, 9, 9, 6, 6, 9, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110082426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9102024-01-24T19:05:46.842Z{"msg":"[0]R generation[0..12]: [7, 3, 6, 6, 7, 7, 4, 4, 7, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110155784Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9112024-01-24T19:05:46.842Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110232639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9122024-01-24T19:05:46.842Z{"msg":"[1]R flush_numbers[0..12]: [9, 3, 8, 8, 9, 9, 6, 6, 9, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110279149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9132024-01-24T19:05:46.842Z{"msg":"[1]R generation[0..12]: [7, 3, 6, 6, 7, 7, 4, 4, 7, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110353577Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161Wait for a query_work_queue command to finish before sending IO
9142024-01-24T19:05:46.842Z,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9152024-01-24T19:05:46.842Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110438424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9162024-01-24T19:05:46.842Z{"msg":"[2]R flush_numbers[0..12]: [9, 3, 8, 8, 9, 9, 6, 6, 9, 1, 2, 6]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110515307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9172024-01-24T19:05:46.842Z{"msg":"[2]R generation[0..12]: [7, 3, 6, 6, 7, 7, 4, 4, 7, 1, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110573788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9182024-01-24T19:05:46.843Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110601678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9192024-01-24T19:05:46.843Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110644765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9202024-01-24T19:05:46.844Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110671291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9212024-01-24T19:05:46.844Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.11069279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9222024-01-24T19:05:46.844Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110744786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9232024-01-24T19:05:46.844Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110773771Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9242024-01-24T19:05:46.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110820409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9252024-01-24T19:05:46.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110845447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9262024-01-24T19:05:46.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.11086723Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9272024-01-24T19:05:46.844Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110915346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9282024-01-24T19:05:46.844Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110939016Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9292024-01-24T19:05:46.844Z{"msg":"822b36b3-f723-4cf6-b428-a5220af6ab97 is now active with session: acdf9ab9-a9e4-43cf-9756-ca8246b0563a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110960841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9302024-01-24T19:05:46.844Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.110980442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161,"session_id":"acdf9ab9-a9e4-43cf-9756-ca8246b0563a"}
9312024-01-24T19:05:46.844Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:45.111000574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1161}
9322024-01-24T19:05:46.844ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9332024-01-24T19:05:46.844ZLoading write count information from file "/tmp/test_repair_verify.data"
9342024-01-24T19:05:46.845ZRun Repair workload
9352024-01-24T19:05:46.845Z01/30 Write block 338 len 32768 data: 3 3 3 3 3 3 3 3
9362024-01-24T19:05:46.845Z02/30 Write block 377 len 12288 data: 2 2 2
9372024-01-24T19:05:46.845Z03/30 Read block 44 len 12288
9382024-01-24T19:05:47.257Z04/30 Read block 480 len 36864
9392024-01-24T19:05:47.257Z05/30 Write block 201 len 12288 data: 2 2 2
9402024-01-24T19:05:47.257Z06/30 Read block 377 len 28672
9412024-01-24T19:05:47.257Z07/30 Read block 547 len 20480
9422024-01-24T19:05:47.257Z08/30 Write block 472 len 8192 data: 2 2
9432024-01-24T19:05:47.257Z09/30 Read block 271 len 20480
9442024-01-24T19:05:47.257Z10/30 Read block 10 len 40960
9452024-01-24T19:05:47.267Z11/30 Write block 40 len 24576 data: 2 3 3 3 4 4
9462024-01-24T19:05:47.267Z12/30 Write block 592 len 28672 data: 2 2 2 2 2 2 2
9472024-01-24T19:05:47.267Z13/30 Read block 248 len 40960
9482024-01-24T19:05:47.267Z14/30 Write block 121 len 20480 data: 2 2 2 2 2
9492024-01-24T19:05:47.268Z15/30 Write block 480 len 12288 data: 2 2 2
9502024-01-24T19:05:47.268Z16/30 Read block 384 len 32768
9512024-01-24T19:05:47.268Z17/30 Read block 118 len 20480
9522024-01-24T19:05:47.268Z18/30 Read block 535 len 12288
9532024-01-24T19:05:47.268Z19/30 Write block 337 len 36864 data: 4 4 4 4 4 4 4 4 4
9542024-01-24T19:05:47.268Z20/30 Write block 200 len 24576 data: 2 3 3 3 2 2
9552024-01-24T19:05:47.268Z21/30 Read block 522 len 20480
9562024-01-24T19:05:47.268Z22/30 Write block 371 len 4096 data: 2
9572024-01-24T19:05:47.268Z23/30 Read block 164 len 4096
9582024-01-24T19:05:47.268Z24/30 Read block 322 len 12288
9592024-01-24T19:05:47.268Z25/30 Read block 493 len 12288
9602024-01-24T19:05:47.269Z26/30 Read block 560 len 40960
9612024-01-24T19:05:47.269Z27/30 Write block 227 len 8192 data: 4 3
9622024-01-24T19:05:47.269Z28/30 Read block 415 len 20480
9632024-01-24T19:05:47.269Z29/30 Write block 15 len 4096 data: 3
9642024-01-24T19:05:47.269Z30/30 Write block 146 len 36864 data: 3 3 3 3 3 3 3 3 2
9652024-01-24T19:05:47.269Z----------------------------------------------------------------
9662024-01-24T19:05:47.269Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9672024-01-24T19:05:47.269ZStates: Active Active Active
9682024-01-24T19:05:47.269ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9692024-01-24T19:05:47.269Z 1 Acked 1000 Write 8 Done Done Done false
9702024-01-24T19:05:47.269Z 2 Acked 1001 Write 3 Done Done Done false
9712024-01-24T19:05:47.269Z 3 Acked 1002 Read 3 Done Done Done false
9722024-01-24T19:05:47.269Z 4 Acked 1003 Read 9 Done Done Done false
9732024-01-24T19:05:47.269Z 5 Acked 1004 Write 3 Done Done Done false
9742024-01-24T19:05:47.269Z 6 Acked 1005 Read 7 Done Sent Done false
9752024-01-24T19:05:47.269Z 7 Acked 1006 Read 5 Done Sent Done false
9762024-01-24T19:05:47.269Z 8 Acked 1007 Write 2 Done Sent Done false
9772024-01-24T19:05:47.269Z 9 Acked 1008 Read 5 Done Sent Done false
9782024-01-24T19:05:47.269Z 10 Acked 1009 Read 10 Done Sent Done false
9792024-01-24T19:05:47.269Z 11 Acked 1010 Write 6 Done Sent Done false
9802024-01-24T19:05:47.269Z 12 Acked 1011 Write 7 Done Sent Done false
9812024-01-24T19:05:47.269Z 13 Acked 1012 Read 10 Done Sent Done false
9822024-01-24T19:05:47.269Z 14 Acked 1013 Write 5 Done Sent Done false
9832024-01-24T19:05:47.270Z 15 Acked 1014 Write 3 Done Sent Done false
9842024-01-24T19:05:47.270Z 16 Acked 1015 Read 8 Done Sent Done false
9852024-01-24T19:05:47.270Z 17 Acked 1016 Read 5 Done Sent Done false
9862024-01-24T19:05:47.270Z 18 Acked 1017 Read 3 Done Sent Done false
9872024-01-24T19:05:47.270Z 19 Acked 1018 Write 9 Done Sent Done false
9882024-01-24T19:05:47.270Z 20 Acked 1019 Write 6 Done Sent Done false
9892024-01-24T19:05:47.270Z 21 Acked 1020 Read 5 Done Sent Done false
9902024-01-24T19:05:47.270Z 22 Acked 1021 Write 1 Done Sent Done false
9912024-01-24T19:05:47.270Z 23 Acked 1022 Read 1 Done Sent Done false
9922024-01-24T19:05:47.270Z 24 Acked 1023 Read 3 Done Sent Done false
9932024-01-24T19:05:47.270Z 25 Acked 1024 Read 3 Done Sent Done false
9942024-01-24T19:05:47.270Z 26 Acked 1025 Read 10 Done Sent Done false
9952024-01-24T19:05:47.270Z 27 Acked 1026 Write 2 Done Sent Done false
9962024-01-24T19:05:47.270Z 28 Acked 1027 Read 5 Done Sent Done false
9972024-01-24T19:05:47.270Z 29 Acked 1028 Write 1 Done Sent Done false
9982024-01-24T19:05:47.270Z 30 Acked 1029 Write 9 Sent Sent Sent false
9992024-01-24T19:05:47.270Z STATES DS:0 DS:1 DS:2 TOTAL
10002024-01-24T19:05:47.270Z New 0 0 0 0
10012024-01-24T19:05:47.270Z Sent 1 25 1 27
10022024-01-24T19:05:47.270Z Done 29 5 29 63
10032024-01-24T19:05:47.270Z Skipped 0 0 0 0
10042024-01-24T19:05:47.270Z Error 0 0 0 0
10052024-01-24T19:05:47.270ZLast Flush: 0 0 0
10062024-01-24T19:05:47.271ZDownstairs last five completed:
10072024-01-24T19:05:47.271ZUpstairs last five completed: 30 29 28 27 26
10082024-01-24T19:05:47.271ZWrote out file "/tmp/test_repair_verify.data"
10092024-01-24T19:05:47.271Z
10102024-01-24T19:05:47.271ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10112024-01-24T19:05:47.271ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10122024-01-24T19:05:47.271Z 0 000-019 7 7 7 9 9 9 T F T <---
10132024-01-24T19:05:47.271Z 1 020-039 3 3 3 3 3 3 F F F
10142024-01-24T19:05:47.271Z 2 040-059 6 6 6 8 8 8 T F T <---
10152024-01-24T19:05:47.271Z 3 060-079 6 6 6 8 8 8 F F F
10162024-01-24T19:05:47.271Z 4 080-099 7 7 7 9 9 9 F F F
10172024-01-24T19:05:47.271Z 5 100-119 7 7 7 9 9 9 F F F
10182024-01-24T19:05:47.271Z 6 120-139 4 4 4 6 6 6 T F T <---
10192024-01-24T19:05:47.271Z 7 140-159 4 4 4 6 6 6 T F T <---
10202024-01-24T19:05:47.271Z 8 160-179 7 7 7 9 9 9 F F F
10212024-01-24T19:05:47.271Z 9 180-199 1 1 1 1 1 1 F F F
10222024-01-24T19:05:47.272Z 10 200-219 2 2 2 2 2 2 T T T
10232024-01-24T19:05:47.272Z 11 220-239 4 4 4 6 6 6 T F T <---
10242024-01-24T19:05:47.272Z 12 240-259 6 6 6 8 8 8 F F F
10252024-01-24T19:05:47.272Z 13 260-279 1 1 1 1 1 1 F F F
10262024-01-24T19:05:47.272Z 14 280-299 5 5 5 7 7 7 F F F
10272024-01-24T19:05:47.272Z 15 300-319 4 4 4 4 4 4 F F F
10282024-01-24T19:05:47.273Z 16 320-339 4 4 4 6 6 6 T T T
10292024-01-24T19:05:47.273Z 17 340-359 6 6 6 8 8 8 T T T
10302024-01-24T19:05:47.273Z 18 360-379 1 1 1 1 1 1 T T T
10312024-01-24T19:05:47.273Z 19 380-399 7 7 7 9 9 9 F F F
10322024-01-24T19:05:47.273Z 20 400-419 6 6 6 8 8 8 F F F
10332024-01-24T19:05:47.273Z 21 420-439 2 2 2 2 2 2 F F F
10342024-01-24T19:05:47.273Z 22 440-459 7 7 7 9 9 9 F F F
10352024-01-24T19:05:47.273Z 23 460-479 7 7 7 9 9 9 T F T <---
10362024-01-24T19:05:47.273Z 24 480-499 3 3 3 3 3 3 T F T <---
10372024-01-24T19:05:47.273Z 25 500-519 3 3 3 3 3 3 F F F
10382024-01-24T19:05:47.277Z 26 520-539 5 5 5 7 7 7 F F F
10392024-01-24T19:05:47.277Z 27 540-559 7 7 7 9 9 9 F F F
10402024-01-24T19:05:47.277Z 28 560-579 1 1 1 1 1 1 F F F
10412024-01-24T19:05:47.277Z 29 580-599 1 1 1 1 1 1 T F T <---
10422024-01-24T19:05:47.277ZMax gen: 7, Max flush: 9
10432024-01-24T19:05:47.277ZError: Difference in extent metadata found!
10442024-01-24T19:05:47.277ZA Difference in extent metadata is expected here
10452024-01-24T19:05:47.277ZOn loop 4
10462024-01-24T19:05:47.278Z
10472024-01-24T19:05:47.280ZVerifying data now
10482024-01-24T19:05:51.663ZNo change after verify
10492024-01-24T19:05:51.663ZLoop: 4 Downstairs dump after verify (and repair):
10502024-01-24T19:05:51.685ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10512024-01-24T19:05:51.686Z 0 000-019 8 8 8 10 10 10 F F F
10522024-01-24T19:05:51.690Z 1 020-039 3 3 3 3 3 3 F F F
10532024-01-24T19:05:51.690Z 2 040-059 8 8 8 10 10 10 F F F
10542024-01-24T19:05:51.691Z 3 060-079 6 6 6 8 8 8 F F F
10552024-01-24T19:05:51.691Z 4 080-099 7 7 7 9 9 9 F F F
10562024-01-24T19:05:51.691Z 5 100-119 7 7 7 9 9 9 F F F
10572024-01-24T19:05:51.691Z 6 120-139 8 8 8 10 10 10 F F F
10582024-01-24T19:05:51.691Z 7 140-159 8 8 8 10 10 10 F F F
10592024-01-24T19:05:51.691Z 8 160-179 7 7 7 9 9 9 F F F
10602024-01-24T19:05:51.691Z 9 180-199 1 1 1 1 1 1 F F F
10612024-01-24T19:05:51.691Z 10 200-219 8 8 8 10 10 10 F F F
10622024-01-24T19:05:51.691Z 11 220-239 8 8 8 10 10 10 F F F
10632024-01-24T19:05:51.691Z 12 240-259 6 6 6 8 8 8 F F F
10642024-01-24T19:05:51.691Z 13 260-279 1 1 1 1 1 1 F F F
10652024-01-24T19:05:51.691Z 14 280-299 5 5 5 7 7 7 F F F
10662024-01-24T19:05:51.691Z 15 300-319 4 4 4 4 4 4 F F F
10672024-01-24T19:05:51.691Z 16 320-339 8 8 8 10 10 10 F F F
10682024-01-24T19:05:51.691Z 17 340-359 8 8 8 10 10 10 F F F
10692024-01-24T19:05:51.691Z 18 360-379 8 8 8 10 10 10 F F F
10702024-01-24T19:05:51.691Z 19 380-399 7 7 7 9 9 9 F F F
10712024-01-24T19:05:51.691Z 20 400-419 6 6 6 8 8 8 F F F
10722024-01-24T19:05:51.691Z 21 420-439 2 2 2 2 2 2 F F F
10732024-01-24T19:05:51.692Z 22 440-459 7 7 7 9 9 9 F F F
10742024-01-24T19:05:51.692Z 23 460-479 8 8 8 10 10 10 F F F
10752024-01-24T19:05:51.692Z 24 480-499 8 8 8 10 10 10 F F F
10762024-01-24T19:05:51.692Z 25 500-519 3 3 3 3 3 3 F F F
10772024-01-24T19:05:51.693Z 26 520-539 5 5 5 7 7 7 F F F
10782024-01-24T19:05:51.693Z 27 540-559 7 7 7 9 9 9 F F F
10792024-01-24T19:05:51.693Z 28 560-579 1 1 1 1 1 1 F F F
10802024-01-24T19:05:51.693Z 29 580-599 8 8 8 10 10 10 F F F
10812024-01-24T19:05:51.693ZMax gen: 8, Max flush: 10
10822024-01-24T19:05:51.693Z
10832024-01-24T19:05:51.693ZBegin loop 5
10842024-01-24T19:05:51.693ZDownstairs to restart: 2
10852024-01-24T19:05:51.719Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.37463172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169}
10862024-01-24T19:05:51.719Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375185572Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169}
10872024-01-24T19:05:51.719Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375233579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169}
10882024-01-24T19:05:51.723Z{"msg":"Crucible e4855484-9ced-4064-add5-b1219e975d07 has session id: 082ee829-0330-44f3-98cd-19b18d35e420","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375291444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10892024-01-24T19:05:51.723Z{"msg":"Upstairs opts: Upstairs UUID: e4855484-9ced-4064-add5-b1219e975d07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375377597Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10902024-01-24T19:05:51.723Z{"msg":"Crucible runtime is spawned
10912024-01-24T19:05:51.723ZCrucible stats registered with UUID: e4855484-9ced-4064-add5-b1219e975d07","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375460564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10922024-01-24T19:05:51.723Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375548708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169}
10932024-01-24T19:05:51.723Z{"msg":"e4855484-9ced-4064-add5-b1219e975d07 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375760218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10942024-01-24T19:05:51.723Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375825513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10952024-01-24T19:05:51.724Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.375854835Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10962024-01-24T19:05:51.725Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.37592949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10972024-01-24T19:05:51.725Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376008756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10982024-01-24T19:05:51.725Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376131397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
10992024-01-24T19:05:51.725Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376183711Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11002024-01-24T19:05:51.725Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.37626339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11012024-01-24T19:05:51.725Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376309894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11022024-01-24T19:05:51.725Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376383424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11032024-01-24T19:05:51.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376430882Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11042024-01-24T19:05:51.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.376474551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11052024-01-24T19:05:51.725Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.37651111Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11062024-01-24T19:05:51.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.377584612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11072024-01-24T19:05:51.725Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.377696264Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11082024-01-24T19:05:51.725Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.377906471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11092024-01-24T19:05:51.725Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.377976324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11102024-01-24T19:05:51.726Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.378429262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11112024-01-24T19:05:51.726Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.3785413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11122024-01-24T19:05:51.731Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.378665114Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11132024-01-24T19:05:51.731Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.378974933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11142024-01-24T19:05:51.731Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.379484701Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11152024-01-24T19:05:51.731Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.379576099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11162024-01-24T19:05:51.732Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.379645576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11172024-01-24T19:05:51.732Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.379743777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11182024-01-24T19:05:51.732Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.38006558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11192024-01-24T19:05:51.733Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.38013831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11202024-01-24T19:05:51.733Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.380176195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11212024-01-24T19:05:51.733Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.380699366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11222024-01-24T19:05:51.733Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.380807141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11232024-01-24T19:05:51.733Z{"msg":"[0]R flush_numbers[0..12]: [10, 3, 10, 8, 9, 9, 10, 10, 9, 1, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.380854174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11242024-01-24T19:05:51.733Z{"msg":"[0]R generation[0..12]: [8, 3, 8, 6, 7, 7, 8, 8, 7, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.380927674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11252024-01-24T19:05:51.733Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381002013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11262024-01-24T19:05:51.734Z{"msg":"[1]R flush_numbers[0..12]: [10, 3, 10, 8, 9, 9, 10, 10, 9, 1, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381079278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11272024-01-24T19:05:51.734Z{"msg":"Wait for a query_work_queue command to finish before sending IO
11282024-01-24T19:05:51.734Z[1]R generation[0..12]: [8, 3, 8, 6, 7, 7, 8, 8, 7, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381183763Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11292024-01-24T19:05:51.734Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381267111Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11302024-01-24T19:05:51.734Z{"msg":"[2]R flush_numbers[0..12]: [10, 3, 10, 8, 9, 9, 10, 10, 9, 1, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.38134904Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11312024-01-24T19:05:51.734Z{"msg":"[2]R generation[0..12]: [8, 3, 8, 6, 7, 7, 8, 8, 7, 1, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381428203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11322024-01-24T19:05:51.734Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381504768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11332024-01-24T19:05:51.734Z{"msg":"Max found gen is 9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381549912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11342024-01-24T19:05:51.734Z{"msg":"Generation requested: 10 >= found:9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381624953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11352024-01-24T19:05:51.734Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381698105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11362024-01-24T19:05:51.735Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381771575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11372024-01-24T19:05:51.735Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.381846604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11382024-01-24T19:05:51.735Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11392024-01-24T19:05:51.735Z,"time":"2024-01-24T19:05:50.381921884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11402024-01-24T19:05:51.735Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.382002146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11412024-01-24T19:05:51.736Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.38204031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11422024-01-24T19:05:51.736Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.382064422Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11432024-01-24T19:05:51.736Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.382108895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11442024-01-24T19:05:51.736Z{"msg":"e4855484-9ced-4064-add5-b1219e975d07 is now active with session: 082ee829-0330-44f3-98cd-19b18d35e420","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.382134334Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11452024-01-24T19:05:51.736Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.38217515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169,"session_id":"082ee829-0330-44f3-98cd-19b18d35e420"}
11462024-01-24T19:05:51.736Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:50.382198791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1169}
11472024-01-24T19:05:51.736ZLoading write count information from file "/tmp/test_repair_verify.data"
11482024-01-24T19:05:51.736ZRun Repair workload
11492024-01-24T19:05:51.736Z01/30 Flush
11502024-01-24T19:05:51.736Z02/30 Write block 45 len 32768 data: 5 3 2 2 2 2 2 2
11512024-01-24T19:05:51.741Z03/30 Write block 383 len 8192 data: 2 2
11522024-01-24T19:05:51.741Z04/30 Read block 358 len 16384
11532024-01-24T19:05:51.741Z05/30 Read block 178 len 32768
11542024-01-24T19:05:51.741Z06/30 Read block 269 len 4096
11552024-01-24T19:05:51.741Z07/30 Write block 288 len 12288 data: 2 3 3
11562024-01-24T19:05:51.741Z08/30 Flush
11572024-01-24T19:05:51.741Z09/30 Write block 134 len 4096 data: 2
11582024-01-24T19:05:51.741Z10/30 Read block 505 len 24576
11592024-01-24T19:05:51.744Z11/30 Read block 517 len 24576
11602024-01-24T19:05:51.745Z12/30 Write block 496 len 16384 data: 2 2 3 3
11612024-01-24T19:05:51.751Z13/30 Write block 504 len 16384 data: 2 2 2 2
11622024-01-24T19:05:51.751Z14/30 Write block 53 len 4096 data: 2
11632024-01-24T19:05:51.751Z15/30 Write block 176 len 16384 data: 2 2 2 2
11642024-01-24T19:05:51.751Z16/30 Write block 119 len 32768 data: 2 2 3 3 3 3 3 2
11652024-01-24T19:05:51.751Z17/30 Write block 358 len 28672 data: 2 2 2 2 2 2 2
11662024-01-24T19:05:51.751Z18/30 Read block 219 len 24576
11672024-01-24T19:05:51.755Z19/30 Read block 13 len 4096
11682024-01-24T19:05:51.755Z20/30 Read block 310 len 4096
11692024-01-24T19:05:51.755Z21/30 Write block 490 len 12288 data: 2 2 2
11702024-01-24T19:05:51.757Z22/30 Read block 127 len 12288
11712024-01-24T19:05:51.760Z23/30 Flush
11722024-01-24T19:05:51.763Z24/30 Flush
11732024-01-24T19:05:51.769Z25/30 Write block 281 len 8192 data: 3 3
11742024-01-24T19:05:51.769Z26/30 Read block 37 len 36864
11752024-01-24T19:05:51.769Z27/30 Write block 385 len 8192 data: 2 3
11762024-01-24T19:05:51.769Z28/30 Write block 174 len 4096 data: 3
11772024-01-24T19:05:51.769Z29/30 Write block 184 len 20480 data: 2 2 2 2 2
11782024-01-24T19:05:51.770Z30/30 Read block 147 len 16384
11792024-01-24T19:05:51.777Z----------------------------------------------------------------
11802024-01-24T19:05:51.777Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:29
11812024-01-24T19:05:51.777ZStates: Active Active Active
11822024-01-24T19:05:51.777ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11832024-01-24T19:05:51.777Z 2 Acked 1001 Write 8 Done Done Done false
11842024-01-24T19:05:51.778Z 3 Acked 1002 Write 2 Done Done Done false
11852024-01-24T19:05:51.778Z 4 Acked 1003 Read 4 Done Done Sent false
11862024-01-24T19:05:51.778Z 5 Acked 1004 Read 8 Done Done Sent false
11872024-01-24T19:05:51.778Z 6 Acked 1005 Read 1 Done Done Sent false
11882024-01-24T19:05:51.778Z 7 Acked 1006 Write 3 Done Done Sent false
11892024-01-24T19:05:51.778Z 8 Acked 1007 Flush 0 Done Done Sent false
11902024-01-24T19:05:51.778Z 9 Acked 1008 Write 1 Done Done Sent false
11912024-01-24T19:05:51.779Z 10 Acked 1009 Read 6 Done Done Sent false
11922024-01-24T19:05:51.779Z 11 Acked 1010 Read 6 Done Done Sent false
11932024-01-24T19:05:51.779Z 12 Acked 1011 Write 4 Done Done Sent false
11942024-01-24T19:05:51.779Z 13 Acked 1012 Write 4 Done Done Sent false
11952024-01-24T19:05:51.779Z 14 Acked 1013 Write 1 Done Done Sent false
11962024-01-24T19:05:51.780Z 15 Acked 1014 Write 4 Done Done Sent false
11972024-01-24T19:05:51.780Z 16 Acked 1015 Write 8 Done Done Sent false
11982024-01-24T19:05:51.780Z 17 Acked 1016 Write 7 Done Done Sent false
11992024-01-24T19:05:51.781Z 18 Acked 1017 Read 6 Done Done Sent false
12002024-01-24T19:05:51.781Z 19 Acked 1018 Read 1 Done Done Sent false
12012024-01-24T19:05:51.781Z 20 Acked 1019 Read 1 Done Done Sent false
12022024-01-24T19:05:51.781Z 21 Acked 1020 Write 3 Done Done Sent false
12032024-01-24T19:05:51.782Z 22 Acked 1021 Read 3 Done Done Sent false
12042024-01-24T19:05:51.782Z 23 Acked 1022 Flush 0 Done Done Sent false
12052024-01-24T19:05:51.782Z 24 Acked 1023 Flush 0 Done Done Sent false
12062024-01-24T19:05:51.782Z 25 Acked 1024 Write 2 Done Done Sent false
12072024-01-24T19:05:51.782Z 26 Acked 1025 Read 9 Done Done Sent false
12082024-01-24T19:05:51.782Z 27 Acked 1026 Write 2 Done Done Sent false
12092024-01-24T19:05:51.783Z 28 Acked 1027 Write 1 Done Done Sent false
12102024-01-24T19:05:51.783Z 29 Acked 1028 Write 5 Done Done Sent false
12112024-01-24T19:05:51.783Z 30 Acked 1029 Read 4 Done Sent Sent false
12122024-01-24T19:05:51.783Z STATES DS:0 DS:1 DS:2 TOTAL
12132024-01-24T19:05:51.783Z New 0 0 0 0
12142024-01-24T19:05:51.784Z Sent 0 1 27 28
12152024-01-24T19:05:51.784Z Done 29 28 2 59
12162024-01-24T19:05:51.784Z Skipped 0 0 0 0
12172024-01-24T19:05:51.784Z Error 0 0 0 0
12182024-01-24T19:05:51.784ZLast Flush: 1023 1023 1000
12192024-01-24T19:05:51.785ZDownstairs last five completed: 1000
12202024-01-24T19:05:51.786ZUpstairs last five completed: 30 29 28 27 26
12212024-01-24T19:05:51.786ZWrote out file "/tmp/test_repair_verify.data"
12222024-01-24T19:05:51.793Z
12232024-01-24T19:05:51.793ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12242024-01-24T19:05:51.813ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12252024-01-24T19:05:51.813Z 0 000-019 8 8 8 10 10 10 F F F
12262024-01-24T19:05:51.813Z 1 020-039 3 3 3 3 3 3 F F F
12272024-01-24T19:05:51.813Z 2 040-059 10 10 8 13 13 10 F F T <---
12282024-01-24T19:05:51.813Z 3 060-079 6 6 6 8 8 8 F F F
12292024-01-24T19:05:51.814Z 4 080-099 7 7 7 9 9 9 F F F
12302024-01-24T19:05:51.814Z 5 100-119 10 10 7 13 13 9 F F F <---
12312024-01-24T19:05:51.814Z 6 120-139 10 10 8 13 13 10 F F F <---
12322024-01-24T19:05:51.822Z 7 140-159 8 8 8 10 10 10 F F F
12332024-01-24T19:05:51.822Z 8 160-179 10 10 7 13 13 9 T T F <---
12342024-01-24T19:05:51.823Z 9 180-199 1 1 1 1 1 1 T T F <---
12352024-01-24T19:05:51.823Z 10 200-219 8 8 8 10 10 10 F F F
12362024-01-24T19:05:51.823Z 11 220-239 8 8 8 10 10 10 F F F
12372024-01-24T19:05:51.823Z 12 240-259 6 6 6 8 8 8 F F F
12382024-01-24T19:05:51.823Z 13 260-279 1 1 1 1 1 1 F F F
12392024-01-24T19:05:51.823Z 14 280-299 10 10 5 12 12 7 T T F <---
12402024-01-24T19:05:51.823Z 15 300-319 4 4 4 4 4 4 F F F
12412024-01-24T19:05:51.824Z 16 320-339 8 8 8 10 10 10 F F F
12422024-01-24T19:05:51.824Z 17 340-359 10 10 8 13 13 10 F F F <---
12432024-01-24T19:05:51.824Z 18 360-379 10 10 8 13 13 10 F F F <---
12442024-01-24T19:05:51.825Z 19 380-399 10 10 7 12 12 9 T T T <---
12452024-01-24T19:05:51.825Z 20 400-419 6 6 6 8 8 8 F F F
12462024-01-24T19:05:51.825Z 21 420-439 2 2 2 2 2 2 F F F
12472024-01-24T19:05:51.826Z 22 440-459 7 7 7 9 9 9 F F F
12482024-01-24T19:05:51.826Z 23 460-479 8 8 8 10 10 10 F F F
12492024-01-24T19:05:51.826Z 24 480-499 10 10 8 13 13 10 F F F <---
12502024-01-24T19:05:51.826Z 25 500-519 10 10 3 13 13 3 F F F <---
12512024-01-24T19:05:51.826Z 26 520-539 5 5 5 7 7 7 F F F
12522024-01-24T19:05:51.827Z 27 540-559 7 7 7 9 9 9 F F F
12532024-01-24T19:05:51.827Z 28 560-579 1 1 1 1 1 1 F F F
12542024-01-24T19:05:51.827Z 29 580-599 8 8 8 10 10 10 F F F
12552024-01-24T19:05:51.827ZMax gen: 10, Max flush: 13
12562024-01-24T19:05:51.827ZError: Difference in extent metadata found!
12572024-01-24T19:05:51.828ZA Difference in extent metadata is expected here
12582024-01-24T19:05:51.828ZOn loop 5
12592024-01-24T19:05:51.828Z
12602024-01-24T19:05:51.829ZVerifying data now
12612024-01-24T19:05:57.835ZNo change after verify
12622024-01-24T19:05:57.835ZLoop: 5 Downstairs dump after verify (and repair):
12632024-01-24T19:05:57.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12642024-01-24T19:05:57.851Z 0 000-019 8 8 8 10 10 10 F F F
12652024-01-24T19:05:57.851Z 1 020-039 3 3 3 3 3 3 F F F
12662024-01-24T19:05:57.851Z 2 040-059 10 10 10 13 13 13 F F F
12672024-01-24T19:05:57.851Z 3 060-079 6 6 6 8 8 8 F F F
12682024-01-24T19:05:57.851Z 4 080-099 7 7 7 9 9 9 F F F
12692024-01-24T19:05:57.851Z 5 100-119 10 10 10 13 13 13 F F F
12702024-01-24T19:05:57.860Z 6 120-139 10 10 10 13 13 13 F F F
12712024-01-24T19:05:57.860Z 7 140-159 8 8 8 10 10 10 F F F
12722024-01-24T19:05:57.860Z 8 160-179 11 11 11 14 14 14 F F F
12732024-01-24T19:05:57.860Z 9 180-199 11 11 11 14 14 14 F F F
12742024-01-24T19:05:57.860Z 10 200-219 8 8 8 10 10 10 F F F
12752024-01-24T19:05:57.860Z 11 220-239 8 8 8 10 10 10 F F F
12762024-01-24T19:05:57.861Z 12 240-259 6 6 6 8 8 8 F F F
12772024-01-24T19:05:57.861Z 13 260-279 1 1 1 1 1 1 F F F
12782024-01-24T19:05:57.861Z 14 280-299 11 11 11 14 14 14 F F F
12792024-01-24T19:05:57.861Z 15 300-319 4 4 4 4 4 4 F F F
12802024-01-24T19:05:57.861Z 16 320-339 8 8 8 10 10 10 F F F
12812024-01-24T19:05:57.861Z 17 340-359 10 10 10 13 13 13 F F F
12822024-01-24T19:05:57.861Z 18 360-379 10 10 10 13 13 13 F F F
12832024-01-24T19:05:57.861Z 19 380-399 11 11 11 14 14 14 F F F
12842024-01-24T19:05:57.861Z 20 400-419 6 6 6 8 8 8 F F F
12852024-01-24T19:05:57.861Z 21 420-439 2 2 2 2 2 2 F F F
12862024-01-24T19:05:57.861Z 22 440-459 7 7 7 9 9 9 F F F
12872024-01-24T19:05:57.861Z 23 460-479 8 8 8 10 10 10 F F F
12882024-01-24T19:05:57.861Z 24 480-499 10 10 10 13 13 13 F F F
12892024-01-24T19:05:57.862Z 25 500-519 10 10 10 13 13 13 F F F
12902024-01-24T19:05:57.862Z 26 520-539 5 5 5 7 7 7 F F F
12912024-01-24T19:05:57.862Z 27 540-559 7 7 7 9 9 9 F F F
12922024-01-24T19:05:57.862Z 28 560-579 1 1 1 1 1 1 F F F
12932024-01-24T19:05:57.862Z 29 580-599 8 8 8 10 10 10 F F F
12942024-01-24T19:05:57.863ZMax gen: 11, Max flush: 14
12952024-01-24T19:05:57.863Z
12962024-01-24T19:05:57.863ZBegin loop 6
12972024-01-24T19:05:57.863ZDownstairs to restart: 1
12982024-01-24T19:05:57.912Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.519291445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177}
12992024-01-24T19:05:57.912Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.51980828Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177}
13002024-01-24T19:05:57.912Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.51983933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177}
13012024-01-24T19:05:57.912Z{"msg":"Crucible fa03f00a-c008-47f0-a8fb-fd414ce118cf has session id: 6f7d34fd-9ca5-4877-9807-228363ba8f33","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.5198884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13022024-01-24T19:05:57.912Z{"msg":"Upstairs opts: Upstairs UUID: fa03f00a-c008-47f0-a8fb-fd414ce118cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.519928882Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13032024-01-24T19:05:57.912Z{"msg":"Crucible stats registered with UUID: fa03f00a-c008-47f0-a8fb-fd414ce118cf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.519953534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13042024-01-24T19:05:57.913ZCrucible runtime is spawned
13052024-01-24T19:05:57.913Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.520239743Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177}
13062024-01-24T19:05:57.913Z{"msg":"fa03f00a-c008-47f0-a8fb-fd414ce118cf active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.520406701Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13072024-01-24T19:05:57.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.520475009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13082024-01-24T19:05:57.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.52063236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13092024-01-24T19:05:57.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.520790506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13102024-01-24T19:05:57.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.52096654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13112024-01-24T19:05:57.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521165715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13122024-01-24T19:05:57.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521198257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13132024-01-24T19:05:57.913Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521301486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13142024-01-24T19:05:57.913Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.52135251Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13152024-01-24T19:05:57.913Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521439766Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13162024-01-24T19:05:57.913Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521468413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13172024-01-24T19:05:57.913Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521497536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13182024-01-24T19:05:57.913Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.521538616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13192024-01-24T19:05:57.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.522408307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13202024-01-24T19:05:57.913Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.522479091Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13212024-01-24T19:05:57.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.522634681Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13222024-01-24T19:05:57.913Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.522690391Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13232024-01-24T19:05:57.913Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.523306129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13242024-01-24T19:05:57.913Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.523380321Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13252024-01-24T19:05:57.913Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.523490349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13262024-01-24T19:05:57.913Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.523590189Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13272024-01-24T19:05:57.914Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524366716Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13282024-01-24T19:05:57.914Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524452816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13292024-01-24T19:05:57.914Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524499062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13302024-01-24T19:05:57.914Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524634893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13312024-01-24T19:05:57.914Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524698349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13322024-01-24T19:05:57.924Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524836792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13332024-01-24T19:05:57.924Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.524902193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13342024-01-24T19:05:57.924Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525509381Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13352024-01-24T19:05:57.924Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525614541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13362024-01-24T19:05:57.924Z{"msg":"[0]R flush_numbers[0..12]: [10, 3, 13, 8, 9, 13, 13, 10, 14, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525643144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13372024-01-24T19:05:57.928Z{"msg":"[0]R generation[0..12]: [8, 3, 10, 6, 7, 10, 10, 8, 11, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525688409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13382024-01-24T19:05:57.928Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525716429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13392024-01-24T19:05:57.928Z{"msg":"[1]R flush_numbers[0..12]: [10, 3, 13, 8, 9, 13, 13, 10, 14, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525760466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13402024-01-24T19:05:57.928Z{"msg":"[1]R generation[0..12]: [8, 3, 10, 6, 7, 10, 10, 8, 11, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525787107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13412024-01-24T19:05:57.928Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525810915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13422024-01-24T19:05:57.928Z{"msg":"[2]R flush_numbers[0..12]: [10, 3, 13, 8, 9, 13, 13, 10, 14, 14, 10, 10]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13432024-01-24T19:05:57.928Z,"time":"2024-01-24T19:05:56.525834436Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13442024-01-24T19:05:57.928Z{"msg":"[2]R generation[0..12]: [8, 3, 10, 6, 7, 10, 10, 8, 11, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525922887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13452024-01-24T19:05:57.928Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525951154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13462024-01-24T19:05:57.928Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525975461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13472024-01-24T19:05:57.928Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.525998191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13482024-01-24T19:05:57.928Z{"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526019447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13492024-01-24T19:05:57.928Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526040908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13502024-01-24T19:05:57.928Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526086524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13512024-01-24T19:05:57.928Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526111021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13522024-01-24T19:05:57.928Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526155164Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13532024-01-24T19:05:57.928Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526181031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13542024-01-24T19:05:57.928Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526203574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13552024-01-24T19:05:57.928Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526224434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13562024-01-24T19:05:57.928Z{"msg":"fa03f00a-c008-47f0-a8fb-fd414ce118cf is now active with session: 6f7d34fd-9ca5-4877-9807-228363ba8f33","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526246548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13572024-01-24T19:05:57.929Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526287172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177,"session_id":"6f7d34fd-9ca5-4877-9807-228363ba8f33"}
13582024-01-24T19:05:57.929Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:05:56.526311629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1177}
13592024-01-24T19:05:57.929ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13602024-01-24T19:05:57.929ZLoading write count information from file "/tmp/test_repair_verify.data"
13612024-01-24T19:05:57.929ZRun Repair workload
13622024-01-24T19:05:57.929Z01/30 Write block 146 len 36864 data: 4 4 4 4 4 4 4 4 3
13632024-01-24T19:05:57.930Z02/30 Read block 85 len 20480
13642024-01-24T19:05:57.930Z03/30 Read block 21 len 16384
13652024-01-24T19:05:57.930Z04/30 Write block 367 len 16384 data: 2 2 2 2
13662024-01-24T19:05:57.930Z05/30 Read block 578 len 28672
13672024-01-24T19:05:57.930Z06/30 Write block 170 len 28672 data: 3 3 3 3 4 3 3
13682024-01-24T19:05:57.930Z07/30 Write block 472 len 4096 data: 3
13692024-01-24T19:05:57.930Z08/30 Write block 154 len 40960 data: 4 2 2 2 2 2 2 2 2 2
13702024-01-24T19:05:57.930Z09/30 Read block 50 len 4096
13712024-01-24T19:05:57.930Z10/30 Write block 15 len 4096 data: 4
13722024-01-24T19:05:57.930Z11/30 Read block 361 len 36864
13732024-01-24T19:05:57.930Z12/30 Write block 134 len 32768 data: 3 2 2 2 2 2 2 3
13742024-01-24T19:05:57.930Z13/30 Write block 508 len 8192 data: 2 2
13752024-01-24T19:05:57.930Z14/30 Write block 28 len 28672 data: 3 3 2 2 2 2 2
13762024-01-24T19:05:57.930Z15/30 Flush
13772024-01-24T19:05:57.930Z16/30 Write block 492 len 20480 data: 3 2 2 2 3
13782024-01-24T19:05:57.931Z17/30 Write block 241 len 24576 data: 2 2 3 3 3 4
13792024-01-24T19:05:57.931Z18/30 Flush
13802024-01-24T19:05:57.931Z19/30 Write block 279 len 40960 data: 2 2 4 4 3 3 2 2 2 3
13812024-01-24T19:05:57.931Z20/30 Write block 548 len 24576 data: 2 2 2 2 2 2
13822024-01-24T19:05:57.931Z21/30 Write block 31 len 16384 data: 3 3 3 3
13832024-01-24T19:05:57.931Z22/30 Read block 254 len 12288
13842024-01-24T19:05:57.931Z23/30 Write block 139 len 24576 data: 3 3 4 3 3 3
13852024-01-24T19:05:57.931Z24/30 Read block 417 len 16384
13862024-01-24T19:05:57.931Z25/30 Write block 513 len 40960 data: 3 3 3 2 3 3 3 3 3 3
13872024-01-24T19:05:57.931Z26/30 Read block 12 len 16384
13882024-01-24T19:05:57.931Z27/30 Write block 320 len 4096 data: 2
13892024-01-24T19:05:57.931Z28/30 Read block 543 len 28672
13902024-01-24T19:05:57.931Z29/30 Write block 83 len 12288 data: 3 3 3
13912024-01-24T19:05:57.931Z30/30 Read block 313 len 24576
13922024-01-24T19:05:57.936Z----------------------------------------------------------------
13932024-01-24T19:05:57.936Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
13942024-01-24T19:05:57.936ZStates: Active Active Active
13952024-01-24T19:05:57.936ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
13962024-01-24T19:05:57.936Z 1 Acked 1000 Write 9 Done Done Done false
13972024-01-24T19:05:57.936Z 2 Acked 1001 Read 5 Done Done Done false
13982024-01-24T19:05:57.936Z 3 Acked 1002 Read 4 Done Done Done false
13992024-01-24T19:05:57.936Z 4 Acked 1003 Write 4 Done Sent Done false
14002024-01-24T19:05:57.936Z 5 Acked 1004 Read 7 Done Sent Done false
14012024-01-24T19:05:57.936Z 6 Acked 1005 Write 7 Done Sent Done false
14022024-01-24T19:05:57.937Z 7 Acked 1006 Write 1 Done Sent Done false
14032024-01-24T19:05:57.937Z 8 Acked 1007 Write 10 Done Sent Done false
14042024-01-24T19:05:57.937Z 9 Acked 1008 Read 1 Done Sent Done false
14052024-01-24T19:05:57.937Z 10 Acked 1009 Write 1 Done Sent Done false
14062024-01-24T19:05:57.937Z 11 Acked 1010 Read 9 Done Sent Done false
14072024-01-24T19:05:57.937Z 12 Acked 1011 Write 8 Done Sent Done false
14082024-01-24T19:05:57.937Z 13 Acked 1012 Write 2 Done Sent Done false
14092024-01-24T19:05:57.937Z 14 Acked 1013 Write 7 Done Sent Done false
14102024-01-24T19:05:57.937Z 15 Acked 1014 Flush 0 Done Sent Done false
14112024-01-24T19:05:57.937Z 16 Acked 1015 Write 5 Done Sent Done false
14122024-01-24T19:05:57.937Z 17 Acked 1016 Write 6 Done Sent Done false
14132024-01-24T19:05:57.937Z 18 Acked 1017 Flush 0 Done Sent Done false
14142024-01-24T19:05:57.937Z 19 Acked 1018 Write 10 Done Sent Done false
14152024-01-24T19:05:57.937Z 20 Acked 1019 Write 6 Done Sent Done false
14162024-01-24T19:05:57.937Z 21 Acked 1020 Write 4 Done Sent Done false
14172024-01-24T19:05:57.937Z 22 Acked 1021 Read 3 Done Sent Done false
14182024-01-24T19:05:57.937Z 23 Acked 1022 Write 6 Done Sent Done false
14192024-01-24T19:05:57.938Z 24 Acked 1023 Read 4 Done Sent Done false
14202024-01-24T19:05:57.938Z 25 Acked 1024 Write 10 Done Sent Done false
14212024-01-24T19:05:57.938Z 26 Acked 1025 Read 4 Done Sent Done false
14222024-01-24T19:05:57.938Z 27 Acked 1026 Write 1 Done Sent Done false
14232024-01-24T19:05:57.938Z 28 Acked 1027 Read 7 Done Sent Done false
14242024-01-24T19:05:57.938Z 29 Acked 1028 Write 3 Done Sent Done false
14252024-01-24T19:05:57.939Z 30 Acked 1029 Read 6 Done Sent Done false
14262024-01-24T19:05:57.939Z STATES DS:0 DS:1 DS:2 TOTAL
14272024-01-24T19:05:57.939Z New 0 0 0 0
14282024-01-24T19:05:57.939Z Sent 0 27 0 27
14292024-01-24T19:05:57.939Z Done 30 3 30 63
14302024-01-24T19:05:57.939Z Skipped 0 0 0 0
14312024-01-24T19:05:57.939Z Error 0 0 0 0
14322024-01-24T19:05:57.940ZLast Flush: 1017 0 1017
14332024-01-24T19:05:57.940ZDownstairs last five completed:
14342024-01-24T19:05:57.940ZUpstairs last five completed: 30 29 28 27 26
14352024-01-24T19:05:57.940ZWrote out file "/tmp/test_repair_verify.data"
14362024-01-24T19:05:57.940Z
14372024-01-24T19:05:57.940ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14382024-01-24T19:05:57.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14392024-01-24T19:05:57.963Z 0 000-019 12 8 12 15 10 15 F F F <---
14402024-01-24T19:05:57.967Z 1 020-039 12 3 12 15 3 15 T F T <---
14412024-01-24T19:05:57.967Z 2 040-059 10 10 10 13 13 13 F F F
14422024-01-24T19:05:57.967Z 3 060-079 6 6 6 8 8 8 F F F
14432024-01-24T19:05:57.967Z 4 080-099 7 7 7 9 9 9 T F T <---
14442024-01-24T19:05:57.967Z 5 100-119 10 10 10 13 13 13 F F F
14452024-01-24T19:05:57.967Z 6 120-139 12 10 12 15 13 15 T F T <---
14462024-01-24T19:05:57.967Z 7 140-159 12 8 12 15 10 15 T T T <---
14472024-01-24T19:05:57.967Z 8 160-179 12 11 12 15 14 15 F F F <---
14482024-01-24T19:05:57.967Z 9 180-199 11 11 11 14 14 14 F F F
14492024-01-24T19:05:57.967Z 10 200-219 8 8 8 10 10 10 F F F
14502024-01-24T19:05:57.968Z 11 220-239 8 8 8 10 10 10 F F F
14512024-01-24T19:05:57.968Z 12 240-259 12 6 12 16 8 16 F F F <---
14522024-01-24T19:05:57.968Z 13 260-279 1 1 1 1 1 1 T F T <---
14532024-01-24T19:05:57.968Z 14 280-299 11 11 11 14 14 14 T F T <---
14542024-01-24T19:05:57.968Z 15 300-319 4 4 4 4 4 4 F F F
14552024-01-24T19:05:57.968Z 16 320-339 8 8 8 10 10 10 T F T <---
14562024-01-24T19:05:57.970Z 17 340-359 10 10 10 13 13 13 F F F
14572024-01-24T19:05:57.970Z 18 360-379 12 10 12 15 13 15 F F F <---
14582024-01-24T19:05:57.970Z 19 380-399 11 11 11 14 14 14 F F F
14592024-01-24T19:05:57.970Z 20 400-419 6 6 6 8 8 8 F F F
14602024-01-24T19:05:57.970Z 21 420-439 2 2 2 2 2 2 F F F
14612024-01-24T19:05:57.970Z 22 440-459 7 7 7 9 9 9 F F F
14622024-01-24T19:05:57.970Z 23 460-479 12 8 12 15 10 15 F F F <---
14632024-01-24T19:05:57.970Z 24 480-499 12 10 12 16 13 16 F F F <---
14642024-01-24T19:05:57.970Z 25 500-519 12 10 12 15 13 15 T F T <---
14652024-01-24T19:05:57.970Z 26 520-539 5 5 5 7 7 7 T F T <---
14662024-01-24T19:05:57.970Z 27 540-559 7 7 7 9 9 9 T F T <---
14672024-01-24T19:05:57.970Z 28 560-579 1 1 1 1 1 1 F F F
14682024-01-24T19:05:57.970Z 29 580-599 8 8 8 10 10 10 F F F
14692024-01-24T19:05:57.970ZMax gen: 12, Max flush: 16
14702024-01-24T19:05:57.970ZError: Difference in extent metadata found!
14712024-01-24T19:05:57.970ZA Difference in extent metadata is expected here
14722024-01-24T19:05:57.970ZOn loop 6
14732024-01-24T19:05:57.970Z
14742024-01-24T19:05:57.974ZVerifying data now
14752024-01-24T19:06:03.351ZNo change after verify
14762024-01-24T19:06:03.351ZLoop: 6 Downstairs dump after verify (and repair):
14772024-01-24T19:06:03.372ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14782024-01-24T19:06:03.373Z 0 000-019 12 12 12 15 15 15 F F F
14792024-01-24T19:06:03.373Z 1 020-039 13 13 13 17 17 17 F F F
14802024-01-24T19:06:03.373Z 2 040-059 10 10 10 13 13 13 F F F
14812024-01-24T19:06:03.373Z 3 060-079 6 6 6 8 8 8 F F F
14822024-01-24T19:06:03.374Z 4 080-099 13 13 13 17 17 17 F F F
14832024-01-24T19:06:03.374Z 5 100-119 10 10 10 13 13 13 F F F
14842024-01-24T19:06:03.374Z 6 120-139 13 13 13 17 17 17 F F F
14852024-01-24T19:06:03.374Z 7 140-159 13 13 13 17 17 17 F F F
14862024-01-24T19:06:03.374Z 8 160-179 12 12 12 15 15 15 F F F
14872024-01-24T19:06:03.375Z 9 180-199 11 11 11 14 14 14 F F F
14882024-01-24T19:06:03.375Z 10 200-219 8 8 8 10 10 10 F F F
14892024-01-24T19:06:03.375Z 11 220-239 8 8 8 10 10 10 F F F
14902024-01-24T19:06:03.375Z 12 240-259 12 12 12 16 16 16 F F F
14912024-01-24T19:06:03.375Z 13 260-279 13 13 13 17 17 17 F F F
14922024-01-24T19:06:03.382Z 14 280-299 13 13 13 17 17 17 F F F
14932024-01-24T19:06:03.382Z 15 300-319 4 4 4 4 4 4 F F F
14942024-01-24T19:06:03.383Z 16 320-339 13 13 13 17 17 17 F F F
14952024-01-24T19:06:03.383Z 17 340-359 10 10 10 13 13 13 F F F
14962024-01-24T19:06:03.383Z 18 360-379 12 12 12 15 15 15 F F F
14972024-01-24T19:06:03.383Z 19 380-399 11 11 11 14 14 14 F F F
14982024-01-24T19:06:03.384Z 20 400-419 6 6 6 8 8 8 F F F
14992024-01-24T19:06:03.384Z 21 420-439 2 2 2 2 2 2 F F F
15002024-01-24T19:06:03.384Z 22 440-459 7 7 7 9 9 9 F F F
15012024-01-24T19:06:03.384Z 23 460-479 12 12 12 15 15 15 F F F
15022024-01-24T19:06:03.384Z 24 480-499 12 12 12 16 16 16 F F F
15032024-01-24T19:06:03.385Z 25 500-519 13 13 13 17 17 17 F F F
15042024-01-24T19:06:03.386Z 26 520-539 13 13 13 17 17 17 F F F
15052024-01-24T19:06:03.387Z 27 540-559 13 13 13 17 17 17 F F F
15062024-01-24T19:06:03.387Z 28 560-579 1 1 1 1 1 1 F F F
15072024-01-24T19:06:03.387Z 29 580-599 8 8 8 10 10 10 F F F
15082024-01-24T19:06:03.387ZMax gen: 13, Max flush: 17
15092024-01-24T19:06:03.388Z
15102024-01-24T19:06:03.388ZBegin loop 7
15112024-01-24T19:06:03.388ZDownstairs to restart: 0
15122024-01-24T19:06:03.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.062838266Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185}
15132024-01-24T19:06:03.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063307073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185}
15142024-01-24T19:06:03.413Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063339611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185}
15152024-01-24T19:06:03.441Z{"msg":"Crucible ea0b8c3c-cc39-45c6-909b-32bfe704c97f has session id: 8ff31bfd-ffb3-4786-a356-354f2459697d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063385774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15162024-01-24T19:06:03.441Z{"msg":"Upstairs opts: Upstairs UUID: ea0b8c3c-cc39-45c6-909b-32bfe704c97f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063463665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15172024-01-24T19:06:03.442Z{"msg":"Crucible stats registered with UUID: ea0b8c3c-cc39-45c6-909b-32bfe704c97f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063539905Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15182024-01-24T19:06:03.442ZCrucible runtime is spawned
15192024-01-24T19:06:03.444Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.063831947Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185}
15202024-01-24T19:06:03.444Z{"msg":"ea0b8c3c-cc39-45c6-909b-32bfe704c97f active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064084358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15212024-01-24T19:06:03.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064142906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15222024-01-24T19:06:03.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064176797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15232024-01-24T19:06:03.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064239071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15242024-01-24T19:06:03.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064270653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15252024-01-24T19:06:03.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064293406Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15262024-01-24T19:06:03.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.06439585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15272024-01-24T19:06:03.447Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064443861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15282024-01-24T19:06:03.447Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064488978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15292024-01-24T19:06:03.447Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064569465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15302024-01-24T19:06:03.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064647184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15312024-01-24T19:06:03.453Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064752184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15322024-01-24T19:06:03.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.064850652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15332024-01-24T19:06:03.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.066145869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15342024-01-24T19:06:03.454Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.066221233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15352024-01-24T19:06:03.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.066249103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15362024-01-24T19:06:03.454Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.06627751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15372024-01-24T19:06:03.454Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.067350765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15382024-01-24T19:06:03.455Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.06761954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15392024-01-24T19:06:03.455Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068133121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15402024-01-24T19:06:03.455Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068248032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15412024-01-24T19:06:03.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.06840868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15422024-01-24T19:06:03.455Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068494531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15432024-01-24T19:06:03.455Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068567642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15442024-01-24T19:06:03.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068614369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15452024-01-24T19:06:03.455Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068692723Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15462024-01-24T19:06:03.455Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.068734341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15472024-01-24T19:06:03.455Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.069218922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15482024-01-24T19:06:03.455Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.069973988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15492024-01-24T19:06:03.455Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070051126Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15502024-01-24T19:06:03.457Z{"msg":"[0]R flush_numbers[0..12]: [15, 17, 13, 8, 17, 13, 17, 17, 15, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070095983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15512024-01-24T19:06:03.457Z{"msg":"[0]R generation[0..12]: [12, 13, 10, 6, 13, 10, 13, 13, 12, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070140049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15522024-01-24T19:06:03.457Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070184458Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15532024-01-24T19:06:03.457Z{"msg":"[1]R flush_numbers[0..12]: [15, 17, 13, 8, 17, 13, 17, 17, 15, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070241609Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15542024-01-24T19:06:03.457Z{"msg":"[1]R generation[0..12]: [12, 13, 10, 6, 13, 10, 13, 13, 12, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070287544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15552024-01-24T19:06:03.457Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.07034543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15562024-01-24T19:06:03.457Z{"msg":"[2]R flush_numbers[0..12]: [15, 17, 13, 8, 17, 13, 17, 17, 15, 14, 10, 10]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15572024-01-24T19:06:03.457Z,"time":"2024-01-24T19:06:02.070391452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15582024-01-24T19:06:03.457Z{"msg":"[2]R generation[0..12]: [12, 13, 10, 6, 13, 10, 13, 13, 12, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.07049588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15592024-01-24T19:06:03.457Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070568646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15602024-01-24T19:06:03.460Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070613768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15612024-01-24T19:06:03.461Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070656197Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15622024-01-24T19:06:03.461Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070696544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15632024-01-24T19:06:03.461Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.07076628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15642024-01-24T19:06:03.461Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070810793Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15652024-01-24T19:06:03.461Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070872202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15662024-01-24T19:06:03.461Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070919641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15672024-01-24T19:06:03.461Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.070975388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15682024-01-24T19:06:03.461Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.071018381Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15692024-01-24T19:06:03.461Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.071075982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15702024-01-24T19:06:03.462Z{"msg":"ea0b8c3c-cc39-45c6-909b-32bfe704c97f is now active with session: 8ff31bfd-ffb3-4786-a356-354f2459697d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.071117815Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15712024-01-24T19:06:03.462Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15722024-01-24T19:06:03.462Z,"time":"2024-01-24T19:06:02.071180672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185,"session_id":"8ff31bfd-ffb3-4786-a356-354f2459697d"}
15732024-01-24T19:06:03.463Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:02.071221638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1185}
15742024-01-24T19:06:03.463ZLoading write count information from file "/tmp/test_repair_verify.data"
15752024-01-24T19:06:03.463ZRun Repair workload
15762024-01-24T19:06:03.463Z01/30 Write block 250 len 4096 data: 2
15772024-01-24T19:06:03.463Z02/30 Read block 542 len 40960
15782024-01-24T19:06:03.884Z03/30 Write block 167 len 40960 data: 2 3 3 4 4 4 4 5 4 4
15792024-01-24T19:06:03.885Z04/30 Read block 251 len 32768
15802024-01-24T19:06:03.885Z05/30 Read block 589 len 4096
15812024-01-24T19:06:03.885Z06/30 Write block 85 len 16384 data: 4 3 3 2
15822024-01-24T19:06:03.885Z07/30 Read block 550 len 28672
15832024-01-24T19:06:03.885Z08/30 Read block 253 len 32768
15842024-01-24T19:06:03.885Z09/30 Read block 83 len 32768
15852024-01-24T19:06:03.885Z10/30 Read block 314 len 40960
15862024-01-24T19:06:03.885Z11/30 Read block 577 len 12288
15872024-01-24T19:06:03.885Z12/30 Read block 38 len 12288
15882024-01-24T19:06:03.885Z13/30 Read block 544 len 32768
15892024-01-24T19:06:03.885Z14/30 Read block 63 len 16384
15902024-01-24T19:06:03.885Z15/30 Flush
15912024-01-24T19:06:03.886Z16/30 Write block 529 len 36864 data: 3 3 2 2 3 3 3 3 3
15922024-01-24T19:06:03.886Z17/30 Read block 331 len 12288
15932024-01-24T19:06:03.886Z18/30 Flush
15942024-01-24T19:06:03.886Z19/30 Write block 20 len 4096 data: 2
15952024-01-24T19:06:03.886Z20/30 Write block 264 len 40960 data: 2 2 2 2 2 2 2 2 2 2
15962024-01-24T19:06:03.886Z21/30 Read block 391 len 16384
15972024-01-24T19:06:03.886Z22/30 Read block 53 len 16384
15982024-01-24T19:06:03.886Z23/30 Write block 453 len 40960 data: 2 2 2 2 2 3 3 3 3 3
15992024-01-24T19:06:03.886Z24/30 Write block 291 len 16384 data: 3 3 3 3
16002024-01-24T19:06:03.886Z25/30 Read block 12 len 40960
16012024-01-24T19:06:03.886Z26/30 Read block 478 len 36864
16022024-01-24T19:06:03.886Z27/30 Write block 161 len 28672 data: 3 3 3 2 2 2 3
16032024-01-24T19:06:03.886Z28/30 Write block 315 len 36864 data: 2 2 2 2 2 3 2 2 3
16042024-01-24T19:06:03.886Z29/30 Read block 81 len 24576
16052024-01-24T19:06:03.887Z30/30 Read block 130 len 4096
16062024-01-24T19:06:03.887Z----------------------------------------------------------------
16072024-01-24T19:06:03.887Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16082024-01-24T19:06:03.887ZStates: Active Active Active
16092024-01-24T19:06:03.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16102024-01-24T19:06:03.887Z 1 Acked 1000 Write 1 Sent Done Done false
16112024-01-24T19:06:03.887Z 2 Acked 1001 Read 10 Sent Done Done false
16122024-01-24T19:06:03.888Z 3 Acked 1002 Write 10 Sent Done Done false
16132024-01-24T19:06:03.888Z 4 Acked 1003 Read 8 Sent Done Done false
16142024-01-24T19:06:03.888Z 5 Acked 1004 Read 1 Sent Done Done false
16152024-01-24T19:06:03.888Z 6 Acked 1005 Write 4 Sent Done Done false
16162024-01-24T19:06:03.888Z 7 Acked 1006 Read 7 Sent Done Done false
16172024-01-24T19:06:03.888Z 8 Acked 1007 Read 8 Sent Done Done false
16182024-01-24T19:06:03.888Z 9 Acked 1008 Read 8 Sent Done Done false
16192024-01-24T19:06:03.888Z 10 Acked 1009 Read 10 Sent Done Done false
16202024-01-24T19:06:03.890Z 11 Acked 1010 Read 3 Sent Done Done false
16212024-01-24T19:06:03.890Z 12 Acked 1011 Read 3 Sent Done Done false
16222024-01-24T19:06:03.890Z 13 Acked 1012 Read 8 Sent Done Done false
16232024-01-24T19:06:03.890Z 14 Acked 1013 Read 4 Sent Done Done false
16242024-01-24T19:06:03.891Z 15 Acked 1014 Flush 0 Sent Done Done false
16252024-01-24T19:06:03.891Z 16 Acked 1015 Write 9 Sent Done Done false
16262024-01-24T19:06:03.891Z 17 Acked 1016 Read 3 Sent Done Done false
16272024-01-24T19:06:03.891Z 18 Acked 1017 Flush 0 Sent Done Done false
16282024-01-24T19:06:03.891Z 19 Acked 1018 Write 1 Sent Done Done false
16292024-01-24T19:06:03.891Z 20 Acked 1019 Write 10 Sent Done Done false
16302024-01-24T19:06:03.891Z 21 Acked 1020 Read 4 Sent Done Done false
16312024-01-24T19:06:03.892Z 22 Acked 1021 Read 4 Sent Done Done false
16322024-01-24T19:06:03.892Z 23 Acked 1022 Write 10 Sent Done Done false
16332024-01-24T19:06:03.892Z 24 Acked 1023 Write 4 Sent Done Done false
16342024-01-24T19:06:03.892Z 25 Acked 1024 Read 10 Sent Done Done false
16352024-01-24T19:06:03.892Z 26 Acked 1025 Read 9 Sent Done Done false
16362024-01-24T19:06:03.892Z 27 Acked 1026 Write 7 Sent Done Done false
16372024-01-24T19:06:03.892Z 28 Acked 1027 Write 9 Sent Done Done false
16382024-01-24T19:06:03.892Z 29 Acked 1028 Read 6 Sent Done Done false
16392024-01-24T19:06:03.892Z 30 Acked 1029 Read 1 Sent Done Done false
16402024-01-24T19:06:03.892Z STATES DS:0 DS:1 DS:2 TOTAL
16412024-01-24T19:06:03.892Z New 0 0 0 0
16422024-01-24T19:06:03.892Z Sent 30 0 0 30
16432024-01-24T19:06:03.892Z Done 0 30 30 60
16442024-01-24T19:06:03.892Z Skipped 0 0 0 0
16452024-01-24T19:06:03.892Z Error 0 0 0 0
16462024-01-24T19:06:03.892ZLast Flush: 0 1017 1017
16472024-01-24T19:06:03.892ZDownstairs last five completed:
16482024-01-24T19:06:03.893ZUpstairs last five completed: 30 29 28 27 26
16492024-01-24T19:06:03.893ZWrote out file "/tmp/test_repair_verify.data"
16502024-01-24T19:06:03.900Z
16512024-01-24T19:06:03.901ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16522024-01-24T19:06:03.901ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16532024-01-24T19:06:03.901Z 0 000-019 12 12 12 15 15 15 F F F
16542024-01-24T19:06:03.901Z 1 020-039 13 13 13 17 17 17 F T T <---
16552024-01-24T19:06:03.901Z 2 040-059 10 10 10 13 13 13 F F F
16562024-01-24T19:06:03.901Z 3 060-079 6 6 6 8 8 8 F F F
16572024-01-24T19:06:03.901Z 4 080-099 13 14 14 17 18 18 F F F <---
16582024-01-24T19:06:03.901Z 5 100-119 10 10 10 13 13 13 F F F
16592024-01-24T19:06:03.901Z 6 120-139 13 13 13 17 17 17 F F F
16602024-01-24T19:06:03.901Z 7 140-159 13 13 13 17 17 17 F F F
16612024-01-24T19:06:03.901Z 8 160-179 12 14 14 15 18 18 F T T <---
16622024-01-24T19:06:03.901Z 9 180-199 11 11 11 14 14 14 F F F
16632024-01-24T19:06:03.901Z 10 200-219 8 8 8 10 10 10 F F F
16642024-01-24T19:06:03.901Z 11 220-239 8 8 8 10 10 10 F F F
16652024-01-24T19:06:03.901Z 12 240-259 12 14 14 16 18 18 F F F <---
16662024-01-24T19:06:03.901Z 13 260-279 13 13 13 17 17 17 F T T <---
16672024-01-24T19:06:03.901Z 14 280-299 13 13 13 17 17 17 F T T <---
16682024-01-24T19:06:03.901Z 15 300-319 4 4 4 4 4 4 F T T <---
16692024-01-24T19:06:03.901Z 16 320-339 13 13 13 17 17 17 F T T <---
16702024-01-24T19:06:03.901Z 17 340-359 10 10 10 13 13 13 F F F
16712024-01-24T19:06:03.902Z 18 360-379 12 12 12 15 15 15 F F F
16722024-01-24T19:06:03.940Z 19 380-399 11 11 11 14 14 14 F F F
16732024-01-24T19:06:03.940Z 20 400-419 6 6 6 8 8 8 F F F
16742024-01-24T19:06:03.940Z 21 420-439 2 2 2 2 2 2 F F F
16752024-01-24T19:06:03.940Z 22 440-459 7 7 7 9 9 9 F T T <---
16762024-01-24T19:06:03.940Z 23 460-479 12 12 12 15 15 15 F T T <---
16772024-01-24T19:06:03.940Z 24 480-499 12 12 12 16 16 16 F F F
16782024-01-24T19:06:03.940Z 25 500-519 13 13 13 17 17 17 F F F
16792024-01-24T19:06:03.941Z 26 520-539 13 14 14 17 19 19 F F F <---
16802024-01-24T19:06:03.941Z 27 540-559 13 13 13 17 17 17 F F F
16812024-01-24T19:06:03.941Z 28 560-579 1 1 1 1 1 1 F F F
16822024-01-24T19:06:03.941Z 29 580-599 8 8 8 10 10 10 F F F
16832024-01-24T19:06:03.941ZMax gen: 14, Max flush: 19
16842024-01-24T19:06:03.941ZError: Difference in extent metadata found!
16852024-01-24T19:06:03.941ZA Difference in extent metadata is expected here
16862024-01-24T19:06:03.941ZOn loop 7
16872024-01-24T19:06:03.941Z
16882024-01-24T19:06:03.942ZVerifying data now
16892024-01-24T19:06:08.542ZNo change after verify
16902024-01-24T19:06:08.542ZLoop: 7 Downstairs dump after verify (and repair):
16912024-01-24T19:06:08.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16922024-01-24T19:06:08.560Z 0 000-019 12 12 12 15 15 15 F F F
16932024-01-24T19:06:08.564Z 1 020-039 15 15 15 20 20 20 F F F
16942024-01-24T19:06:08.564Z 2 040-059 10 10 10 13 13 13 F F F
16952024-01-24T19:06:08.564Z 3 060-079 6 6 6 8 8 8 F F F
16962024-01-24T19:06:08.564Z 4 080-099 14 14 14 18 18 18 F F F
16972024-01-24T19:06:08.564Z 5 100-119 10 10 10 13 13 13 F F F
16982024-01-24T19:06:08.564Z 6 120-139 13 13 13 17 17 17 F F F
16992024-01-24T19:06:08.565Z 7 140-159 13 13 13 17 17 17 F F F
17002024-01-24T19:06:08.565Z 8 160-179 15 15 15 20 20 20 F F F
17012024-01-24T19:06:08.565Z 9 180-199 11 11 11 14 14 14 F F F
17022024-01-24T19:06:08.565Z 10 200-219 8 8 8 10 10 10 F F F
17032024-01-24T19:06:08.565Z 11 220-239 8 8 8 10 10 10 F F F
17042024-01-24T19:06:08.565Z 12 240-259 14 14 14 18 18 18 F F F
17052024-01-24T19:06:08.565Z 13 260-279 15 15 15 20 20 20 F F F
17062024-01-24T19:06:08.565Z 14 280-299 15 15 15 20 20 20 F F F
17072024-01-24T19:06:08.565Z 15 300-319 15 15 15 20 20 20 F F F
17082024-01-24T19:06:08.566Z 16 320-339 15 15 15 20 20 20 F F F
17092024-01-24T19:06:08.566Z 17 340-359 10 10 10 13 13 13 F F F
17102024-01-24T19:06:08.566Z 18 360-379 12 12 12 15 15 15 F F F
17112024-01-24T19:06:08.567Z 19 380-399 11 11 11 14 14 14 F F F
17122024-01-24T19:06:08.567Z 20 400-419 6 6 6 8 8 8 F F F
17132024-01-24T19:06:08.567Z 21 420-439 2 2 2 2 2 2 F F F
17142024-01-24T19:06:08.567Z 22 440-459 15 15 15 20 20 20 F F F
17152024-01-24T19:06:08.567Z 23 460-479 15 15 15 20 20 20 F F F
17162024-01-24T19:06:08.567Z 24 480-499 12 12 12 16 16 16 F F F
17172024-01-24T19:06:08.567Z 25 500-519 13 13 13 17 17 17 F F F
17182024-01-24T19:06:08.567Z 26 520-539 14 14 14 19 19 19 F F F
17192024-01-24T19:06:08.567Z 27 540-559 13 13 13 17 17 17 F F F
17202024-01-24T19:06:08.567Z 28 560-579 1 1 1 1 1 1 F F F
17212024-01-24T19:06:08.567Z 29 580-599 8 8 8 10 10 10 F F F
17222024-01-24T19:06:08.567ZMax gen: 15, Max flush: 20
17232024-01-24T19:06:08.570Z
17242024-01-24T19:06:08.570ZBegin loop 8
17252024-01-24T19:06:08.571ZDownstairs to restart: 0
17262024-01-24T19:06:08.597Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.250587264Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193}
17272024-01-24T19:06:08.597Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251053181Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193}
17282024-01-24T19:06:08.619Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251112428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193}
17292024-01-24T19:06:08.620Z{"msg":"Crucible 41cfc591-b574-4504-b3e7-24ca6826687a has session id: 2abad71b-02ea-4af1-a44a-0853065fbf87","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251226897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17302024-01-24T19:06:08.620Z{"msg":"Upstairs opts: Upstairs UUID: 41cfc591-b574-4504-b3e7-24ca6826687a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251332287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17312024-01-24T19:06:08.620Z{"msg":"Crucible stats registered with UUID: 41cfc591-b574-4504-b3e7-24ca6826687a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251362933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17322024-01-24T19:06:08.620ZCrucible runtime is spawned
17332024-01-24T19:06:08.620Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251748348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193}
17342024-01-24T19:06:08.620Z{"msg":"41cfc591-b574-4504-b3e7-24ca6826687a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251847901Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17352024-01-24T19:06:08.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251907954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17362024-01-24T19:06:08.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251936249Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17372024-01-24T19:06:08.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.251991974Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17382024-01-24T19:06:08.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.25207843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17392024-01-24T19:06:08.620Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252124468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17402024-01-24T19:06:08.620Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252169121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17412024-01-24T19:06:08.621Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252206621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17422024-01-24T19:06:08.621Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.25224983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17432024-01-24T19:06:08.621Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252326039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17442024-01-24T19:06:08.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252371624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17452024-01-24T19:06:08.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252633195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17462024-01-24T19:06:08.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.252684836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17472024-01-24T19:06:08.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.253722036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17482024-01-24T19:06:08.621Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.253804344Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17492024-01-24T19:06:08.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.254008336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17502024-01-24T19:06:08.622Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.254098749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17512024-01-24T19:06:08.623Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.2542198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17522024-01-24T19:06:08.623Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.254256537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17532024-01-24T19:06:08.623Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.255110578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17542024-01-24T19:06:08.623Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.255183306Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17552024-01-24T19:06:08.623Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.255426485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17562024-01-24T19:06:08.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256154806Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17572024-01-24T19:06:08.623Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256228977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17582024-01-24T19:06:08.623Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256258233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17592024-01-24T19:06:08.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256279578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17602024-01-24T19:06:08.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256323842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17612024-01-24T19:06:08.623Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.25634988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17622024-01-24T19:06:08.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256387618Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17632024-01-24T19:06:08.624Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256433705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17642024-01-24T19:06:08.624Z{"msg":"[0]R flush_numbers[0..12]: [15, 20, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256460742Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17652024-01-24T19:06:08.624Z{"msg":"[0]R generation[0..12]: [12, 15, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256485432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17662024-01-24T19:06:08.624Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256529684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17672024-01-24T19:06:08.624Z{"msg":"[1]R flush_numbers[0..12]: [15, 20, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256556203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17682024-01-24T19:06:08.624Z{"msg":"[1]R generation[0..12]: [12, 15, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256599261Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17692024-01-24T19:06:08.624Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256626586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17702024-01-24T19:06:08.624Z{"msg":"[2]R flush_numbers[0..12]: [15, 20, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256671328Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17712024-01-24T19:06:08.624Z{"msg":"[2]R generation[0..12]: [12, 15, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256697434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17722024-01-24T19:06:08.624ZWait for a query_work_queue command to finish before sending IO
17732024-01-24T19:06:08.624Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256745346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17742024-01-24T19:06:08.624Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256827765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17752024-01-24T19:06:08.624Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256882543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17762024-01-24T19:06:08.624Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256908034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17772024-01-24T19:06:08.624Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256959192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17782024-01-24T19:06:08.624Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.256994293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17792024-01-24T19:06:08.624Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257015794Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17802024-01-24T19:06:08.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257037244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17812024-01-24T19:06:08.625Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.2570587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17822024-01-24T19:06:08.625Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257114503Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17832024-01-24T19:06:08.625Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257139057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17842024-01-24T19:06:08.625Z{"msg":"41cfc591-b574-4504-b3e7-24ca6826687a is now active with session: 2abad71b-02ea-4af1-a44a-0853065fbf87","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257161007Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17852024-01-24T19:06:08.625Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257181297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193,"session_id":"2abad71b-02ea-4af1-a44a-0853065fbf87"}
17862024-01-24T19:06:08.625Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:07.257230405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1193}
17872024-01-24T19:06:08.625ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17882024-01-24T19:06:08.626ZLoading write count information from file "/tmp/test_repair_verify.data"
17892024-01-24T19:06:08.626ZRun Repair workload
17902024-01-24T19:06:08.626Z01/30 Write block 556 len 40960 data: 2 2 2 2 2 2 2 2 2 2
17912024-01-24T19:06:08.629Z02/30 Write block 290 len 20480 data: 4 4 4 4 4
17922024-01-24T19:06:08.629Z03/30 Write block 311 len 32768 data: 2 2 2 2 3 3 3 3
17932024-01-24T19:06:08.629Z04/30 Read block 42 len 36864
17942024-01-24T19:06:08.629Z05/30 Read block 217 len 16384
17952024-01-24T19:06:08.630Z06/30 Read block 311 len 28672
17962024-01-24T19:06:08.630Z07/30 Read block 580 len 32768
17972024-01-24T19:06:08.630Z08/30 Write block 391 len 28672 data: 3 3 4 4 4 4 3
17982024-01-24T19:06:08.630Z09/30 Read block 177 len 4096
17992024-01-24T19:06:08.630Z10/30 Flush
18002024-01-24T19:06:08.630Z11/30 Write block 410 len 16384 data: 3 3 2 3
18012024-01-24T19:06:08.630Z12/30 Read block 580 len 36864
18022024-01-24T19:06:08.630Z13/30 Read block 62 len 8192
18032024-01-24T19:06:08.630Z14/30 Read block 356 len 16384
18042024-01-24T19:06:08.630Z15/30 Read block 535 len 40960
18052024-01-24T19:06:08.637Z16/30 Write block 28 len 28672 data: 4 4 3 4 4 4 4
18062024-01-24T19:06:08.637Z17/30 Write block 542 len 12288 data: 3 3 3
18072024-01-24T19:06:08.637Z18/30 Write block 547 len 24576 data: 2 3 3 3 3 3
18082024-01-24T19:06:08.638Z19/30 Flush
18092024-01-24T19:06:08.649Z20/30 Write block 273 len 8192 data: 3 2
18102024-01-24T19:06:08.649Z21/30 Read block 203 len 12288
18112024-01-24T19:06:08.651Z22/30 Read block 422 len 20480
18122024-01-24T19:06:08.651Z23/30 Read block 119 len 8192
18132024-01-24T19:06:08.651Z24/30 Write block 482 len 36864 data: 3 2 2 2 2 2 2 2 3
18142024-01-24T19:06:08.651Z25/30 Write block 264 len 16384 data: 3 3 3 3
18152024-01-24T19:06:08.651Z26/30 Write block 279 len 20480 data: 3 3 5 5 4
18162024-01-24T19:06:08.673Z27/30 Read block 514 len 16384
18172024-01-24T19:06:08.673Z28/30 Read block 557 len 36864
18182024-01-24T19:06:08.674Z29/30 Write block 447 len 24576 data: 2 2 2 2 2 2
18192024-01-24T19:06:08.674Z30/30 Write block 458 len 12288 data: 4 4 4
18202024-01-24T19:06:08.674Z----------------------------------------------------------------
18212024-01-24T19:06:08.674Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18222024-01-24T19:06:08.674ZStates: Active Active Active
18232024-01-24T19:06:08.674ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18242024-01-24T19:06:08.675Z 1 Acked 1000 Write 10 Done Done Done false
18252024-01-24T19:06:08.675Z 2 Acked 1001 Write 5 Done Done Done false
18262024-01-24T19:06:08.675Z 3 Acked 1002 Write 8 Done Done Done false
18272024-01-24T19:06:08.675Z 4 Acked 1003 Read 9 Done Done Done false
18282024-01-24T19:06:08.675Z 5 Acked 1004 Read 4 Done Done Done false
18292024-01-24T19:06:08.675Z 6 Acked 1005 Read 7 Done Done Done false
18302024-01-24T19:06:08.675Z 7 Acked 1006 Read 8 Sent Done Done false
18312024-01-24T19:06:08.676Z 8 Acked 1007 Write 7 Sent Done Done false
18322024-01-24T19:06:08.676Z 9 Acked 1008 Read 1 Sent Done Done false
18332024-01-24T19:06:08.676Z 10 Acked 1009 Flush 0 Sent Done Done false
18342024-01-24T19:06:08.676Z 11 Acked 1010 Write 4 Sent Done Done false
18352024-01-24T19:06:08.676Z 12 Acked 1011 Read 9 Sent Done Done false
18362024-01-24T19:06:08.676Z 13 Acked 1012 Read 2 Sent Done Done false
18372024-01-24T19:06:08.676Z 14 Acked 1013 Read 4 Sent Done Done false
18382024-01-24T19:06:08.677Z 15 Acked 1014 Read 10 Sent Done Done false
18392024-01-24T19:06:08.677Z 16 Acked 1015 Write 7 Sent Done Done false
18402024-01-24T19:06:08.677Z 17 Acked 1016 Write 3 Sent Done Done false
18412024-01-24T19:06:08.677Z 18 Acked 1017 Write 6 Sent Done Done false
18422024-01-24T19:06:08.677Z 19 Acked 1018 Flush 0 Sent Done Done false
18432024-01-24T19:06:08.678Z 20 Acked 1019 Write 2 Sent Done Done false
18442024-01-24T19:06:08.678Z 21 Acked 1020 Read 3 Sent Done Done false
18452024-01-24T19:06:08.679Z 22 Acked 1021 Read 5 Sent Done Done false
18462024-01-24T19:06:08.679Z 23 Acked 1022 Read 2 Sent Done Done false
18472024-01-24T19:06:08.679Z 24 Acked 1023 Write 9 Sent Done Done false
18482024-01-24T19:06:08.679Z 25 Acked 1024 Write 4 Sent Done Done false
18492024-01-24T19:06:08.679Z 26 Acked 1025 Write 5 Sent Done Done false
18502024-01-24T19:06:08.679Z 27 Acked 1026 Read 4 Sent Done Done false
18512024-01-24T19:06:08.679Z 28 Acked 1027 Read 9 Sent Done Done false
18522024-01-24T19:06:08.680Z 29 Acked 1028 Write 6 Sent Done Done false
18532024-01-24T19:06:08.680Z 30 Acked 1029 Write 3 Sent Sent Sent false
18542024-01-24T19:06:08.680Z STATES DS:0 DS:1 DS:2 TOTAL
18552024-01-24T19:06:08.680Z New 0 0 0 0
18562024-01-24T19:06:08.680Z Sent 24 1 1 26
18572024-01-24T19:06:08.680Z Done 6 29 29 64
18582024-01-24T19:06:08.680Z Skipped 0 0 0 0
18592024-01-24T19:06:08.681Z Error 0 0 0 0
18602024-01-24T19:06:08.681ZLast Flush: 0 1018 1018
18612024-01-24T19:06:08.681ZDownstairs last five completed:
18622024-01-24T19:06:08.681ZUpstairs last five completed: 30 29 28 27 26
18632024-01-24T19:06:08.681ZWrote out file "/tmp/test_repair_verify.data"
18642024-01-24T19:06:08.681Z
18652024-01-24T19:06:08.688ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18662024-01-24T19:06:08.693ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18672024-01-24T19:06:08.696Z 0 000-019 12 12 12 15 15 15 F F F
18682024-01-24T19:06:08.696Z 1 020-039 15 16 16 20 22 22 F F F <---
18692024-01-24T19:06:08.696Z 2 040-059 10 10 10 13 13 13 F F F
18702024-01-24T19:06:08.697Z 3 060-079 6 6 6 8 8 8 F F F
18712024-01-24T19:06:08.697Z 4 080-099 14 14 14 18 18 18 F F F
18722024-01-24T19:06:08.697Z 5 100-119 10 10 10 13 13 13 F F F
18732024-01-24T19:06:08.697Z 6 120-139 13 13 13 17 17 17 F F F
18742024-01-24T19:06:08.697Z 7 140-159 13 13 13 17 17 17 F F F
18752024-01-24T19:06:08.698Z 8 160-179 15 15 15 20 20 20 F F F
18762024-01-24T19:06:08.698Z 9 180-199 11 11 11 14 14 14 F F F
18772024-01-24T19:06:08.698Z 10 200-219 8 8 8 10 10 10 F F F
18782024-01-24T19:06:08.698Z 11 220-239 8 8 8 10 10 10 F F F
18792024-01-24T19:06:08.698Z 12 240-259 14 14 14 18 18 18 F F F
18802024-01-24T19:06:08.698Z 13 260-279 15 15 15 20 20 20 F T T <---
18812024-01-24T19:06:08.698Z 14 280-299 15 16 16 20 21 21 T T T <---
18822024-01-24T19:06:08.698Z 15 300-319 15 16 16 20 21 21 T F F <---
18832024-01-24T19:06:08.699Z 16 320-339 15 15 15 20 20 20 F F F
18842024-01-24T19:06:08.699Z 17 340-359 10 10 10 13 13 13 F F F
18852024-01-24T19:06:08.699Z 18 360-379 12 12 12 15 15 15 F F F
18862024-01-24T19:06:08.699Z 19 380-399 11 16 16 14 21 21 F F F <---
18872024-01-24T19:06:08.699Z 20 400-419 6 16 16 8 22 22 F F F <---
18882024-01-24T19:06:08.699Z 21 420-439 2 2 2 2 2 2 F F F
18892024-01-24T19:06:08.699Z 22 440-459 15 15 15 20 20 20 F T T <---
18902024-01-24T19:06:08.700Z 23 460-479 15 15 15 20 20 20 F T T <---
18912024-01-24T19:06:08.700Z 24 480-499 12 12 12 16 16 16 F T T <---
18922024-01-24T19:06:08.701Z 25 500-519 13 13 13 17 17 17 F F F
18932024-01-24T19:06:08.701Z 26 520-539 14 14 14 19 19 19 F F F
18942024-01-24T19:06:08.701Z 27 540-559 13 16 16 17 22 22 T F F <---
18952024-01-24T19:06:08.701Z 28 560-579 1 16 16 1 21 21 T F F <---
18962024-01-24T19:06:08.701Z 29 580-599 8 8 8 10 10 10 F F F
18972024-01-24T19:06:08.701ZMax gen: 16, Max flush: 22
18982024-01-24T19:06:08.702ZError: Difference in extent metadata found!
18992024-01-24T19:06:08.702ZA Difference in extent metadata is expected here
19002024-01-24T19:06:08.702ZOn loop 8
19012024-01-24T19:06:08.702Z
19022024-01-24T19:06:08.707ZVerifying data now
19032024-01-24T19:06:14.712ZNo change after verify
19042024-01-24T19:06:14.713ZLoop: 8 Downstairs dump after verify (and repair):
19052024-01-24T19:06:14.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19062024-01-24T19:06:14.732Z 0 000-019 12 12 12 15 15 15 F F F
19072024-01-24T19:06:14.732Z 1 020-039 16 16 16 22 22 22 F F F
19082024-01-24T19:06:14.732Z 2 040-059 10 10 10 13 13 13 F F F
19092024-01-24T19:06:14.742Z 3 060-079 6 6 6 8 8 8 F F F
19102024-01-24T19:06:14.742Z 4 080-099 14 14 14 18 18 18 F F F
19112024-01-24T19:06:14.742Z 5 100-119 10 10 10 13 13 13 F F F
19122024-01-24T19:06:14.742Z 6 120-139 13 13 13 17 17 17 F F F
19132024-01-24T19:06:14.743Z 7 140-159 13 13 13 17 17 17 F F F
19142024-01-24T19:06:14.743Z 8 160-179 15 15 15 20 20 20 F F F
19152024-01-24T19:06:14.744Z 9 180-199 11 11 11 14 14 14 F F F
19162024-01-24T19:06:14.744Z 10 200-219 8 8 8 10 10 10 F F F
19172024-01-24T19:06:14.744Z 11 220-239 8 8 8 10 10 10 F F F
19182024-01-24T19:06:14.744Z 12 240-259 14 14 14 18 18 18 F F F
19192024-01-24T19:06:14.744Z 13 260-279 17 17 17 23 23 23 F F F
19202024-01-24T19:06:14.744Z 14 280-299 17 17 17 23 23 23 F F F
19212024-01-24T19:06:14.745Z 15 300-319 16 16 16 21 21 21 F F F
19222024-01-24T19:06:14.746Z 16 320-339 15 15 15 20 20 20 F F F
19232024-01-24T19:06:14.746Z 17 340-359 10 10 10 13 13 13 F F F
19242024-01-24T19:06:14.746Z 18 360-379 12 12 12 15 15 15 F F F
19252024-01-24T19:06:14.746Z 19 380-399 16 16 16 21 21 21 F F F
19262024-01-24T19:06:14.746Z 20 400-419 16 16 16 22 22 22 F F F
19272024-01-24T19:06:14.747Z 21 420-439 2 2 2 2 2 2 F F F
19282024-01-24T19:06:14.747Z 22 440-459 17 17 17 23 23 23 F F F
19292024-01-24T19:06:14.747Z 23 460-479 17 17 17 23 23 23 F F F
19302024-01-24T19:06:14.747Z 24 480-499 17 17 17 23 23 23 F F F
19312024-01-24T19:06:14.747Z 25 500-519 13 13 13 17 17 17 F F F
19322024-01-24T19:06:14.747Z 26 520-539 14 14 14 19 19 19 F F F
19332024-01-24T19:06:14.747Z 27 540-559 16 16 16 22 22 22 F F F
19342024-01-24T19:06:14.748Z 28 560-579 16 16 16 21 21 21 F F F
19352024-01-24T19:06:14.748Z 29 580-599 8 8 8 10 10 10 F F F
19362024-01-24T19:06:14.748ZMax gen: 17, Max flush: 23
19372024-01-24T19:06:14.748Z
19382024-01-24T19:06:14.748ZBegin loop 9
19392024-01-24T19:06:14.748ZDownstairs to restart: 1
19402024-01-24T19:06:14.771Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.413427161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201}
19412024-01-24T19:06:14.771Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.413880623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201}
19422024-01-24T19:06:14.771Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41394727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201}
19432024-01-24T19:06:14.771Z{"msg":"Crucible 14e756bd-f43a-451b-bc41-759e9225384f has session id: 06093d46-129f-413f-b08c-594ce82b3f86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.4140164Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19442024-01-24T19:06:14.772Z{"msg":"Upstairs opts: Upstairs UUID: 14e756bd-f43a-451b-bc41-759e9225384f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
19452024-01-24T19:06:14.772Z,"time":"2024-01-24T19:06:13.414103548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19462024-01-24T19:06:14.772Z{"msg":"Crucible stats registered with UUID: 14e756bd-f43a-451b-bc41-759e9225384f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.414315667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19472024-01-24T19:06:14.773Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.414346229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201}
19482024-01-24T19:06:14.774Z{"msg":"14e756bd-f43a-451b-bc41-759e9225384f active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.414644927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19492024-01-24T19:06:14.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.414720355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19502024-01-24T19:06:14.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.414906453Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19512024-01-24T19:06:14.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415332683Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19522024-01-24T19:06:14.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415447765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19532024-01-24T19:06:14.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415496794Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19542024-01-24T19:06:14.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415541398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19552024-01-24T19:06:14.795Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415602973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19562024-01-24T19:06:14.795Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415631858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19572024-01-24T19:06:14.798Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415655221Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19582024-01-24T19:06:14.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415731194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19592024-01-24T19:06:14.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415760089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19602024-01-24T19:06:14.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.415790654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19612024-01-24T19:06:14.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.416879207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19622024-01-24T19:06:14.799Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.416923068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19632024-01-24T19:06:14.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.417038495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19642024-01-24T19:06:14.799Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.417128664Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19652024-01-24T19:06:14.799Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41721195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19662024-01-24T19:06:14.799Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41731986Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19672024-01-24T19:06:14.800Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.418042952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19682024-01-24T19:06:14.800Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.418172139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19692024-01-24T19:06:14.800Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.418253685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19702024-01-24T19:06:14.800Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419034063Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19712024-01-24T19:06:14.800Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419100298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19722024-01-24T19:06:14.800Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41912786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19732024-01-24T19:06:14.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419149135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19742024-01-24T19:06:14.801Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419192586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19752024-01-24T19:06:14.801Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419217703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19762024-01-24T19:06:14.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419258878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19772024-01-24T19:06:14.804Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419284458Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19782024-01-24T19:06:14.804Z{"msg":"[0]R flush_numbers[0..12]: [15, 22, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419328387Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19792024-01-24T19:06:14.805Z{"msg":"[0]R generation[0..12]: [12, 16, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419354743Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19802024-01-24T19:06:14.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419398782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19812024-01-24T19:06:14.805Z{"msg":"[1]R flush_numbers[0..12]: [15, 22, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419424885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19822024-01-24T19:06:14.805ZWait for a query_work_queue command to finish before sending IO
19832024-01-24T19:06:14.805Z{"msg":"[1]R generation[0..12]: [12, 16, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419527003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19842024-01-24T19:06:14.805Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419586545Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19852024-01-24T19:06:14.805Z{"msg":"[2]R flush_numbers[0..12]: [15, 22, 13, 8, 18, 13, 17, 17, 20, 14, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419614073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19862024-01-24T19:06:14.805Z{"msg":"[2]R generation[0..12]: [12, 16, 10, 6, 14, 10, 13, 13, 15, 11, 8, 8]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419665332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19872024-01-24T19:06:14.807Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419692116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19882024-01-24T19:06:14.807Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419734403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19892024-01-24T19:06:14.807Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419759699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19902024-01-24T19:06:14.807Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419781223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19912024-01-24T19:06:14.807Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419802555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19922024-01-24T19:06:14.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419824566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19932024-01-24T19:06:14.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419865067Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19942024-01-24T19:06:14.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.419889866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19952024-01-24T19:06:14.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41993674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19962024-01-24T19:06:14.808Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.41996214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19972024-01-24T19:06:14.808Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.420020768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19982024-01-24T19:06:14.808Z{"msg":"14e756bd-f43a-451b-bc41-759e9225384f is now active with session: 06093d46-129f-413f-b08c-594ce82b3f86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.420047287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
19992024-01-24T19:06:14.808Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.420089072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201,"session_id":"06093d46-129f-413f-b08c-594ce82b3f86"}
20002024-01-24T19:06:14.809Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:13.420112626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1201}
20012024-01-24T19:06:14.809ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20022024-01-24T19:06:14.809ZLoading write count information from file "/tmp/test_repair_verify.data"
20032024-01-24T19:06:14.809ZRun Repair workload
20042024-01-24T19:06:14.809Z01/30 Write block 376 len 8192 data: 2 3
20052024-01-24T19:06:14.809Z02/30 Write block 298 len 40960 data: 2 2 2 2 2 2 2 3 3 2
20062024-01-24T19:06:14.809Z03/30 Write block 106 len 32768 data: 2 2 2 2 2 2 2 2
20072024-01-24T19:06:14.809Z04/30 Write block 77 len 20480 data: 2 2 2 2 2
20082024-01-24T19:06:14.810Z05/30 Read block 132 len 36864
20092024-01-24T19:06:14.810Z06/30 Read block 204 len 16384
20102024-01-24T19:06:14.810Z07/30 Write block 216 len 28672 data: 2 2 2 2 2 2 2
20112024-01-24T19:06:14.810Z08/30 Write block 181 len 24576 data: 2 2 2 3 3 3
20122024-01-24T19:06:14.810Z09/30 Read block 414 len 32768
20132024-01-24T19:06:14.810Z10/30 Read block 229 len 36864
20142024-01-24T19:06:14.843Z11/30 Write block 319 len 40960 data: 3 4 3 3 4 3 3 3 3 3
20152024-01-24T19:06:14.844Z12/30 Write block 64 len 8192 data: 2 2
20162024-01-24T19:06:14.844Z13/30 Write block 21 len 32768 data: 2 2 2 3 3 3 3 5
20172024-01-24T19:06:14.844Z14/30 Read block 162 len 28672
20182024-01-24T19:06:14.845Z15/30 Read block 237 len 16384
20192024-01-24T19:06:14.845Z16/30 Read block 504 len 16384
20202024-01-24T19:06:14.845Z17/30 Read block 139 len 40960
20212024-01-24T19:06:14.845Z18/30 Read block 412 len 36864
20222024-01-24T19:06:14.845Z19/30 Flush
20232024-01-24T19:06:14.845Z20/30 Write block 3 len 32768 data: 2 2 2 2 2 3 3 3
20242024-01-24T19:06:14.846Z21/30 Read block 334 len 28672
20252024-01-24T19:06:14.846Z22/30 Write block 552 len 12288 data: 4 3 2
20262024-01-24T19:06:14.846Z23/30 Read block 478 len 20480
20272024-01-24T19:06:14.846Z24/30 Write block 422 len 4096 data: 2
20282024-01-24T19:06:14.846Z25/30 Read block 376 len 16384
20292024-01-24T19:06:14.846Z26/30 Read block 193 len 16384
20302024-01-24T19:06:14.847Z27/30 Write block 81 len 4096 data: 3
20312024-01-24T19:06:14.847Z28/30 Read block 505 len 32768
20322024-01-24T19:06:14.847Z29/30 Write block 279 len 12288 data: 4 4 6
20332024-01-24T19:06:14.847Z30/30 Read block 578 len 36864
20342024-01-24T19:06:14.847Z----------------------------------------------------------------
20352024-01-24T19:06:14.847Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20362024-01-24T19:06:14.847ZStates: Active Active Active
20372024-01-24T19:06:14.848ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20382024-01-24T19:06:14.848Z 1 Acked 1000 Write 2 Done Sent Done false
20392024-01-24T19:06:14.848Z 2 Acked 1001 Write 10 Done Sent Done false
20402024-01-24T19:06:14.848Z 3 Acked 1002 Write 8 Done Sent Done false
20412024-01-24T19:06:14.848Z 4 Acked 1003 Write 5 Done Sent Done false
20422024-01-24T19:06:14.848Z 5 Acked 1004 Read 9 Done Sent Done false
20432024-01-24T19:06:14.848Z 6 Acked 1005 Read 4 Done Sent Done false
20442024-01-24T19:06:14.848Z 7 Acked 1006 Write 7 Done Sent Done false
20452024-01-24T19:06:14.849Z 8 Acked 1007 Write 6 Done Sent Done false
20462024-01-24T19:06:14.849Z 9 Acked 1008 Read 8 Done Sent Done false
20472024-01-24T19:06:14.849Z 10 Acked 1009 Read 9 Done Sent Done false
20482024-01-24T19:06:14.849Z 11 Acked 1010 Write 10 Done Sent Done false
20492024-01-24T19:06:14.849Z 12 Acked 1011 Write 2 Done Sent Done false
20502024-01-24T19:06:14.852Z 13 Acked 1012 Write 8 Done Sent Done false
20512024-01-24T19:06:14.852Z 14 Acked 1013 Read 7 Done Sent Done false
20522024-01-24T19:06:14.852Z 15 Acked 1014 Read 4 Done Sent Done false
20532024-01-24T19:06:14.852Z 16 Acked 1015 Read 4 Done Sent Done false
20542024-01-24T19:06:14.852Z 17 Acked 1016 Read 10 Done Sent Done false
20552024-01-24T19:06:14.853Z 18 Acked 1017 Read 9 Done Sent Done false
20562024-01-24T19:06:14.853Z 19 Acked 1018 Flush 0 Done Sent Done false
20572024-01-24T19:06:14.853Z 20 Acked 1019 Write 8 Done Sent Done false
20582024-01-24T19:06:14.853Z 21 Acked 1020 Read 7 Done Sent Done false
20592024-01-24T19:06:14.853Z 22 Acked 1021 Write 3 Done Sent Done false
20602024-01-24T19:06:14.853Z 23 Acked 1022 Read 5 Done Sent Done false
20612024-01-24T19:06:14.853Z 24 Acked 1023 Write 1 Done Sent Done false
20622024-01-24T19:06:14.854Z 25 Acked 1024 Read 4 Done Sent Done false
20632024-01-24T19:06:14.854Z 26 Acked 1025 Read 4 Done Sent Done false
20642024-01-24T19:06:14.854Z 27 Acked 1026 Write 1 Done Sent Done false
20652024-01-24T19:06:14.854Z 28 Acked 1027 Read 8 Done Sent Done false
20662024-01-24T19:06:14.854Z 29 Acked 1028 Write 3 Done Sent Done false
20672024-01-24T19:06:14.854Z 30 Acked 1029 Read 9 Done Sent Sent false
20682024-01-24T19:06:14.854Z STATES DS:0 DS:1 DS:2 TOTAL
20692024-01-24T19:06:14.855Z New 0 0 0 0
20702024-01-24T19:06:14.855Z Sent 0 30 1 31
20712024-01-24T19:06:14.855Z Done 30 0 29 59
20722024-01-24T19:06:14.855Z Skipped 0 0 0 0
20732024-01-24T19:06:14.855Z Error 0 0 0 0
20742024-01-24T19:06:14.855ZLast Flush: 1018 0 1018
20752024-01-24T19:06:14.855ZDownstairs last five completed:
20762024-01-24T19:06:14.855ZUpstairs last five completed: 30 29 28 27 26
20772024-01-24T19:06:14.855ZWrote out file "/tmp/test_repair_verify.data"
20782024-01-24T19:06:14.946Z
20792024-01-24T19:06:14.947ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20802024-01-24T19:06:14.947ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20812024-01-24T19:06:14.947Z 0 000-019 12 12 12 15 15 15 T F T <---
20822024-01-24T19:06:14.947Z 1 020-039 18 16 18 24 22 24 F F F <---
20832024-01-24T19:06:14.947Z 2 040-059 10 10 10 13 13 13 F F F
20842024-01-24T19:06:14.947Z 3 060-079 18 6 18 24 8 24 F F F <---
20852024-01-24T19:06:14.948Z 4 080-099 18 14 18 24 18 24 T F T <---
20862024-01-24T19:06:14.948Z 5 100-119 18 10 18 24 13 24 F F F <---
20872024-01-24T19:06:14.948Z 6 120-139 13 13 13 17 17 17 F F F
20882024-01-24T19:06:14.948Z 7 140-159 13 13 13 17 17 17 F F F
20892024-01-24T19:06:14.948Z 8 160-179 15 15 15 20 20 20 F F F
20902024-01-24T19:06:14.948Z 9 180-199 18 11 18 24 14 24 F F F <---
20912024-01-24T19:06:14.949Z 10 200-219 18 8 18 24 10 24 F F F <---
20922024-01-24T19:06:14.949Z 11 220-239 18 8 18 24 10 24 F F F <---
20932024-01-24T19:06:14.949Z 12 240-259 14 14 14 18 18 18 F F F
20942024-01-24T19:06:14.949Z 13 260-279 17 17 17 23 23 23 T F T <---
20952024-01-24T19:06:14.949Z 14 280-299 18 17 18 24 23 24 T F T <---
20962024-01-24T19:06:14.949Z 15 300-319 18 16 18 24 21 24 F F F <---
20972024-01-24T19:06:14.949Z 16 320-339 18 15 18 24 20 24 F F F <---
20982024-01-24T19:06:14.950Z 17 340-359 10 10 10 13 13 13 F F F
20992024-01-24T19:06:14.951Z 18 360-379 18 12 18 24 15 24 F F F <---
21002024-01-24T19:06:14.951Z 19 380-399 16 16 16 21 21 21 F F F
21012024-01-24T19:06:14.951Z 20 400-419 16 16 16 22 22 22 F F F
21022024-01-24T19:06:14.951Z 21 420-439 2 2 2 2 2 2 T F T <---
21032024-01-24T19:06:14.952Z 22 440-459 17 17 17 23 23 23 F F F
21042024-01-24T19:06:14.952Z 23 460-479 17 17 17 23 23 23 F F F
21052024-01-24T19:06:14.952Z 24 480-499 17 17 17 23 23 23 F F F
21062024-01-24T19:06:14.952Z 25 500-519 13 13 13 17 17 17 F F F
21072024-01-24T19:06:14.952Z 26 520-539 14 14 14 19 19 19 F F F
21082024-01-24T19:06:14.953Z 27 540-559 16 16 16 22 22 22 T F T <---
21092024-01-24T19:06:14.953Z 28 560-579 16 16 16 21 21 21 F F F
21102024-01-24T19:06:14.953Z 29 580-599 8 8 8 10 10 10 F F F
21112024-01-24T19:06:14.953ZMax gen: 18, Max flush: 24
21122024-01-24T19:06:14.953ZError: Difference in extent metadata found!
21132024-01-24T19:06:14.953ZA Difference in extent metadata is expected here
21142024-01-24T19:06:14.954ZOn loop 9
21152024-01-24T19:06:14.954Z
21162024-01-24T19:06:14.954ZVerifying data now
21172024-01-24T19:06:20.678ZNo change after verify
21182024-01-24T19:06:21.128ZLoop: 9 Downstairs dump after verify (and repair):
21192024-01-24T19:06:21.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21202024-01-24T19:06:21.129Z 0 000-019 19 19 19 25 25 25 F F F
21212024-01-24T19:06:21.130Z 1 020-039 18 18 18 24 24 24 F F F
21222024-01-24T19:06:21.130Z 2 040-059 10 10 10 13 13 13 F F F
21232024-01-24T19:06:21.130Z 3 060-079 18 18 18 24 24 24 F F F
21242024-01-24T19:06:21.130Z 4 080-099 19 19 19 25 25 25 F F F
21252024-01-24T19:06:21.130Z 5 100-119 18 18 18 24 24 24 F F F
21262024-01-24T19:06:21.130Z 6 120-139 13 13 13 17 17 17 F F F
21272024-01-24T19:06:21.130Z 7 140-159 13 13 13 17 17 17 F F F
21282024-01-24T19:06:21.130Z 8 160-179 15 15 15 20 20 20 F F F
21292024-01-24T19:06:21.130Z 9 180-199 18 18 18 24 24 24 F F F
21302024-01-24T19:06:21.130Z 10 200-219 18 18 18 24 24 24 F F F
21312024-01-24T19:06:21.130Z 11 220-239 18 18 18 24 24 24 F F F
21322024-01-24T19:06:21.130Z 12 240-259 14 14 14 18 18 18 F F F
21332024-01-24T19:06:21.130Z 13 260-279 19 19 19 25 25 25 F F F
21342024-01-24T19:06:21.131Z 14 280-299 19 19 19 25 25 25 F F F
21352024-01-24T19:06:21.131Z 15 300-319 18 18 18 24 24 24 F F F
21362024-01-24T19:06:21.131Z 16 320-339 18 18 18 24 24 24 F F F
21372024-01-24T19:06:21.131Z 17 340-359 10 10 10 13 13 13 F F F
21382024-01-24T19:06:21.131Z 18 360-379 18 18 18 24 24 24 F F F
21392024-01-24T19:06:21.131Z 19 380-399 16 16 16 21 21 21 F F F
21402024-01-24T19:06:21.131Z 20 400-419 16 16 16 22 22 22 F F F
21412024-01-24T19:06:21.131Z 21 420-439 19 19 19 25 25 25 F F F
21422024-01-24T19:06:21.131Z 22 440-459 17 17 17 23 23 23 F F F
21432024-01-24T19:06:21.131Z 23 460-479 17 17 17 23 23 23 F F F
21442024-01-24T19:06:21.131Z 24 480-499 17 17 17 23 23 23 F F F
21452024-01-24T19:06:21.131Z 25 500-519 13 13 13 17 17 17 F F F
21462024-01-24T19:06:21.132Z 26 520-539 14 14 14 19 19 19 F F F
21472024-01-24T19:06:21.132Z 27 540-559 19 19 19 25 25 25 F F F
21482024-01-24T19:06:21.132Z 28 560-579 16 16 16 21 21 21 F F F
21492024-01-24T19:06:21.132Z 29 580-599 8 8 8 10 10 10 F F F
21502024-01-24T19:06:21.132ZMax gen: 19, Max flush: 25
21512024-01-24T19:06:21.132Z
21522024-01-24T19:06:21.132ZBegin loop 10
21532024-01-24T19:06:21.132ZDownstairs to restart: 1
21542024-01-24T19:06:21.132Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875171535Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209}
21552024-01-24T19:06:21.132Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875650842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209}
21562024-01-24T19:06:21.133Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875707486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209}
21572024-01-24T19:06:21.133Z{"msg":"Crucible 9db1aef7-ccae-4dcd-b6f5-233afa622562 has session id: 0e68e1f4-7890-4ef0-8a94-91af8bfa25ae","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875754143Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21582024-01-24T19:06:21.133Z{"msg":"Upstairs opts: Upstairs UUID: 9db1aef7-ccae-4dcd-b6f5-233afa622562, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875792459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21592024-01-24T19:06:21.133Z{"msg":"Crucible stats registered with UUID: 9db1aef7-ccae-4dcd-b6f5-233afa622562","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.875821424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21602024-01-24T19:06:21.134ZCrucible runtime is spawned
21612024-01-24T19:06:21.134Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876120976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209}
21622024-01-24T19:06:21.134Z{"msg":"9db1aef7-ccae-4dcd-b6f5-233afa622562 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876246008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21632024-01-24T19:06:21.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876315741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21642024-01-24T19:06:21.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.87634445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21652024-01-24T19:06:21.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.87640493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21662024-01-24T19:06:21.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876481484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21672024-01-24T19:06:21.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876551424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21682024-01-24T19:06:21.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876631649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21692024-01-24T19:06:21.134Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.876709981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21702024-01-24T19:06:21.134Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.8768141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21712024-01-24T19:06:21.136Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.877073556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21722024-01-24T19:06:21.136Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.877113991Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21732024-01-24T19:06:21.136Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.877164702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21742024-01-24T19:06:21.136Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.877218006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21752024-01-24T19:06:21.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878413248Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21762024-01-24T19:06:21.136Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878496072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21772024-01-24T19:06:21.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878570467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21782024-01-24T19:06:21.136Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878650695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21792024-01-24T19:06:21.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878696987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21802024-01-24T19:06:21.136Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.878771103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21812024-01-24T19:06:21.140Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.87969281Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21822024-01-24T19:06:21.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.87985447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21832024-01-24T19:06:21.141Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.880133717Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21842024-01-24T19:06:21.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.88068709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21852024-01-24T19:06:21.141Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.880749076Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21862024-01-24T19:06:21.141Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.880826302Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21872024-01-24T19:06:21.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.880916739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21882024-01-24T19:06:21.141Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881039724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21892024-01-24T19:06:21.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881085485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21902024-01-24T19:06:21.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881124465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21912024-01-24T19:06:21.143Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881173508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21922024-01-24T19:06:21.143Z{"msg":"[0]R flush_numbers[0..12]: [25, 24, 13, 24, 25, 24, 17, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881220591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21932024-01-24T19:06:21.143Z{"msg":"[0]R generation[0..12]: [19, 18, 10, 18, 19, 18, 13, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881267128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21942024-01-24T19:06:21.143Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881314483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21952024-01-24T19:06:21.143Z{"msg":"[1]R flush_numbers[0..12]: [25, 24, 13, 24, 25, 24, 17, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881369708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21962024-01-24T19:06:21.143Z{"msg":"[1]R generation[0..12]: [19, 18, 10, 18, 19, 18, 13, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
21972024-01-24T19:06:21.144Z,"time":"2024-01-24T19:06:18.881440115Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21982024-01-24T19:06:21.144Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881558439Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
21992024-01-24T19:06:21.144Z{"msg":"[2]R flush_numbers[0..12]: [25, 24, 13, 24, 25, 24, 17, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881598772Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22002024-01-24T19:06:21.144Z{"msg":"[2]R generation[0..12]: [19, 18, 10, 18, 19, 18, 13, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881658982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22012024-01-24T19:06:21.144Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881725429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22022024-01-24T19:06:21.144Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881786667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22032024-01-24T19:06:21.144Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881838933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22042024-01-24T19:06:21.145Z{"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881888741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22052024-01-24T19:06:21.145Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881934191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22062024-01-24T19:06:21.145Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.881991867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22072024-01-24T19:06:21.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882046037Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22082024-01-24T19:06:21.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882091877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22092024-01-24T19:06:21.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882137555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22102024-01-24T19:06:21.145ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22112024-01-24T19:06:21.145Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882186297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22122024-01-24T19:06:21.146Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882230779Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22132024-01-24T19:06:21.146Z{"msg":"9db1aef7-ccae-4dcd-b6f5-233afa622562 is now active with session: 0e68e1f4-7890-4ef0-8a94-91af8bfa25ae","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882275646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22142024-01-24T19:06:21.146Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.882318826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209,"session_id":"0e68e1f4-7890-4ef0-8a94-91af8bfa25ae"}
22152024-01-24T19:06:21.146Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:18.88236256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1209}
22162024-01-24T19:06:21.147ZLoading write count information from file "/tmp/test_repair_verify.data"
22172024-01-24T19:06:21.147ZRun Repair workload
22182024-01-24T19:06:21.175ZError: Difference in extent metadata found!
22192024-01-24T19:06:21.175Z01/30 Write block 59 len 4096 data: 3
22202024-01-24T19:06:21.175Z02/30 Read block 410 len 8192
22212024-01-24T19:06:21.175Z03/30 Read block 120 len 40960
22222024-01-24T19:06:21.175Z04/30 Read block 120 len 32768
22232024-01-24T19:06:21.176Z05/30 Read block 450 len 40960
22242024-01-24T19:06:21.176Z06/30 Write block 108 len 8192 data: 3 3
22252024-01-24T19:06:21.176Z07/30 Read block 583 len 8192
22262024-01-24T19:06:21.176Z08/30 Flush
22272024-01-24T19:06:21.176Z09/30 Write block 70 len 24576 data: 2 2 2 2 2 2
22282024-01-24T19:06:21.176Z10/30 Read block 433 len 24576
22292024-01-24T19:06:21.176Z11/30 Write block 24 len 20480 data: 4 4 4 4 6
22302024-01-24T19:06:21.176Z12/30 Flush
22312024-01-24T19:06:21.176Z13/30 Write block 278 len 24576 data: 2 5 5 7 6 5
22322024-01-24T19:06:21.177Z14/30 Read block 387 len 28672
22332024-01-24T19:06:21.177Z15/30 Read block 120 len 12288
22342024-01-24T19:06:21.178Z16/30 Write block 448 len 8192 data: 3 3
22352024-01-24T19:06:21.178Z17/30 Flush
22362024-01-24T19:06:21.178Z18/30 Write block 30 len 16384 data: 4 5 5 5
22372024-01-24T19:06:21.178Z19/30 Read block 25 len 8192
22382024-01-24T19:06:21.178Z20/30 Read block 160 len 20480
22392024-01-24T19:06:21.178Z21/30 Read block 322 len 4096
22402024-01-24T19:06:21.178Z22/30 Read block 27 len 24576
22412024-01-24T19:06:21.178Z23/30 Read block 341 len 36864
22422024-01-24T19:06:21.178Z24/30 Read block 483 len 12288
22432024-01-24T19:06:21.178Z25/30 Write block 547 len 12288 data: 3 4 4
22442024-01-24T19:06:21.179Z26/30 Flush
22452024-01-24T19:06:21.179Z27/30 Write block 118 len 40960 data: 2 3 3 4 4 4 4 4 3 2
22462024-01-24T19:06:21.179Z28/30 Read block 121 len 28672
22472024-01-24T19:06:21.179Z29/30 Read block 134 len 28672
22482024-01-24T19:06:21.179Z30/30 Read block 441 len 36864
22492024-01-24T19:06:21.179Z----------------------------------------------------------------
22502024-01-24T19:06:21.179Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22512024-01-24T19:06:21.179ZStates: Active Active Active
22522024-01-24T19:06:21.179ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22532024-01-24T19:06:21.179Z 1 Acked 1000 Write 1 Done Sent Done false
22542024-01-24T19:06:21.180Z 2 Acked 1001 Read 2 Done Sent Done false
22552024-01-24T19:06:21.180Z 3 Acked 1002 Read 10 Done Sent Done false
22562024-01-24T19:06:21.180Z 4 Acked 1003 Read 8 Done Sent Done false
22572024-01-24T19:06:21.180Z 5 Acked 1004 Read 10 Done Sent Done false
22582024-01-24T19:06:21.181Z 6 Acked 1005 Write 2 Done Sent Done false
22592024-01-24T19:06:21.181Z 7 Acked 1006 Read 2 Done Sent Done false
22602024-01-24T19:06:21.181Z 8 Acked 1007 Flush 0 Done Sent Done false
22612024-01-24T19:06:21.181Z 9 Acked 1008 Write 6 Done Sent Done false
22622024-01-24T19:06:21.181Z 10 Acked 1009 Read 6 Done Sent Done false
22632024-01-24T19:06:21.182Z 11 Acked 1010 Write 5 Done Sent Done false
22642024-01-24T19:06:21.182Z 12 Acked 1011 Flush 0 Done Sent Done false
22652024-01-24T19:06:21.182Z 13 Acked 1012 Write 6 Done Sent Done false
22662024-01-24T19:06:21.182Z 14 Acked 1013 Read 7 Done Sent Done false
22672024-01-24T19:06:21.182Z 15 Acked 1014 Read 3 Done Sent Done false
22682024-01-24T19:06:21.182Z 16 Acked 1015 Write 2 Done Sent Done false
22692024-01-24T19:06:21.183Z 17 Acked 1016 Flush 0 Done Sent Done false
22702024-01-24T19:06:21.183Z 18 Acked 1017 Write 4 Done Sent Done false
22712024-01-24T19:06:21.183Z 19 Acked 1018 Read 2 Done Sent Done false
22722024-01-24T19:06:21.183Z 20 Acked 1019 Read 5 Done Sent Done false
22732024-01-24T19:06:21.183Z 21 Acked 1020 Read 1 Done Sent Done false
22742024-01-24T19:06:21.183Z 22 Acked 1021 Read 6 Done Sent Done false
22752024-01-24T19:06:21.184Z 23 Acked 1022 Read 9 Done Sent Done false
22762024-01-24T19:06:21.184Z 24 Acked 1023 Read 3 Done Sent Done false
22772024-01-24T19:06:21.184Z 25 Acked 1024 Write 3 Done Sent Done false
22782024-01-24T19:06:21.184Z 26 Acked 1025 Flush 0 Done Sent Done false
22792024-01-24T19:06:21.184Z 27 Acked 1026 Write 10 Done Sent Done false
22802024-01-24T19:06:21.184Z 28 Acked 1027 Read 7 Done Sent Done false
22812024-01-24T19:06:21.185Z 29 Acked 1028 Read 7 Done Sent Done false
22822024-01-24T19:06:21.185Z 30 Acked 1029 Read 9 Sent Sent Done false
22832024-01-24T19:06:21.185Z STATES DS:0 DS:1 DS:2 TOTAL
22842024-01-24T19:06:21.185Z New 0 0 0 0
22852024-01-24T19:06:21.186Z Sent 1 30 0 31
22862024-01-24T19:06:21.186Z Done 29 0 30 59
22872024-01-24T19:06:21.186Z Skipped 0 0 0 0
22882024-01-24T19:06:21.186Z Error 0 0 0 0
22892024-01-24T19:06:21.187ZLast Flush: 1025 0 1025
22902024-01-24T19:06:21.187ZDownstairs last five completed:
22912024-01-24T19:06:21.187ZUpstairs last five completed: 30 29 28 27 26
22922024-01-24T19:06:21.187ZWrote out file "/tmp/test_repair_verify.data"
22932024-01-24T19:06:21.188Z
22942024-01-24T19:06:21.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
22952024-01-24T19:06:21.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
22962024-01-24T19:06:21.188Z 0 000-019 19 19 19 25 25 25 F F F
22972024-01-24T19:06:21.188Z 1 020-039 20 18 20 29 24 29 F F F <---
22982024-01-24T19:06:21.188Z 2 040-059 20 10 20 26 13 26 F F F <---
22992024-01-24T19:06:21.189Z 3 060-079 20 18 20 27 24 27 F F F <---
23002024-01-24T19:06:21.189Z 4 080-099 19 19 19 25 25 25 F F F
23012024-01-24T19:06:21.189Z 5 100-119 20 18 20 26 24 26 T F T <---
23022024-01-24T19:06:21.189Z 6 120-139 13 13 13 17 17 17 T F T <---
23032024-01-24T19:06:21.189Z 7 140-159 13 13 13 17 17 17 F F F
23042024-01-24T19:06:21.189Z 8 160-179 15 15 15 20 20 20 F F F
23052024-01-24T19:06:21.189Z 9 180-199 18 18 18 24 24 24 F F F
23062024-01-24T19:06:21.190Z 10 200-219 18 18 18 24 24 24 F F F
23072024-01-24T19:06:21.191Z 11 220-239 18 18 18 24 24 24 F F F
23082024-01-24T19:06:21.191Z 12 240-259 14 14 14 18 18 18 F F F
23092024-01-24T19:06:21.191Z 13 260-279 20 19 20 28 25 28 F F F <---
23102024-01-24T19:06:21.191Z 14 280-299 20 19 20 28 25 28 F F F <---
23112024-01-24T19:06:21.191Z 15 300-319 18 18 18 24 24 24 F F F
23122024-01-24T19:06:21.192Z 16 320-339 18 18 18 24 24 24 F F F
23132024-01-24T19:06:21.192Z 17 340-359 10 10 10 13 13 13 F F F
23142024-01-24T19:06:21.192Z 18 360-379 18 18 18 24 24 24 F F F
23152024-01-24T19:06:21.192Z 19 380-399 16 16 16 21 21 21 F F F
23162024-01-24T19:06:21.192Z 20 400-419 16 16 16 22 22 22 F F F
23172024-01-24T19:06:21.193Z 21 420-439 19 19 19 25 25 25 F F F
23182024-01-24T19:06:21.216Z 22 440-459 20 17 20 28 23 28 F F F <---
23192024-01-24T19:06:21.216Z 23 460-479 17 17 17 23 23 23 F F F
23202024-01-24T19:06:21.216Z 24 480-499 17 17 17 23 23 23 F F F
23212024-01-24T19:06:21.217Z 25 500-519 13 13 13 17 17 17 F F F
23222024-01-24T19:06:21.217Z 26 520-539 14 14 14 19 19 19 F F F
23232024-01-24T19:06:21.217Z 27 540-559 20 19 20 29 25 29 F F F <---
23242024-01-24T19:06:21.217Z 28 560-579 16 16 16 21 21 21 F F F
23252024-01-24T19:06:21.217Z 29 580-599 8 8 8 10 10 10 F F F
23262024-01-24T19:06:21.217ZMax gen: 20, Max flush: 29
23272024-01-24T19:06:21.218ZA Difference in extent metadata is expected here
23282024-01-24T19:06:21.218ZOn loop 10
23292024-01-24T19:06:21.218Z
23302024-01-24T19:06:21.218ZVerifying data now
23312024-01-24T19:06:25.179ZNo change after verify
23322024-01-24T19:06:25.179ZLoop: 10 Downstairs dump after verify (and repair):
23332024-01-24T19:06:25.195ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23342024-01-24T19:06:25.195Z 0 000-019 19 19 19 25 25 25 F F F
23352024-01-24T19:06:25.219Z 1 020-039 20 20 20 29 29 29 F F F
23362024-01-24T19:06:25.219Z 2 040-059 20 20 20 26 26 26 F F F
23372024-01-24T19:06:25.220Z 3 060-079 20 20 20 27 27 27 F F F
23382024-01-24T19:06:25.220Z 4 080-099 19 19 19 25 25 25 F F F
23392024-01-24T19:06:25.220Z 5 100-119 21 21 21 30 30 30 F F F
23402024-01-24T19:06:25.220Z 6 120-139 21 21 21 30 30 30 F F F
23412024-01-24T19:06:25.220Z 7 140-159 13 13 13 17 17 17 F F F
23422024-01-24T19:06:25.220Z 8 160-179 15 15 15 20 20 20 F F F
23432024-01-24T19:06:25.220Z 9 180-199 18 18 18 24 24 24 F F F
23442024-01-24T19:06:25.220Z 10 200-219 18 18 18 24 24 24 F F F
23452024-01-24T19:06:25.220Z 11 220-239 18 18 18 24 24 24 F F F
23462024-01-24T19:06:25.221Z 12 240-259 14 14 14 18 18 18 F F F
23472024-01-24T19:06:25.221Z 13 260-279 20 20 20 28 28 28 F F F
23482024-01-24T19:06:25.221Z 14 280-299 20 20 20 28 28 28 F F F
23492024-01-24T19:06:25.221Z 15 300-319 18 18 18 24 24 24 F F F
23502024-01-24T19:06:25.221Z 16 320-339 18 18 18 24 24 24 F F F
23512024-01-24T19:06:25.221Z 17 340-359 10 10 10 13 13 13 F F F
23522024-01-24T19:06:25.223Z 18 360-379 18 18 18 24 24 24 F F F
23532024-01-24T19:06:25.223Z 19 380-399 16 16 16 21 21 21 F F F
23542024-01-24T19:06:25.223Z 20 400-419 16 16 16 22 22 22 F F F
23552024-01-24T19:06:25.223Z 21 420-439 19 19 19 25 25 25 F F F
23562024-01-24T19:06:25.224Z 22 440-459 20 20 20 28 28 28 F F F
23572024-01-24T19:06:25.224Z 23 460-479 17 17 17 23 23 23 F F F
23582024-01-24T19:06:25.224Z 24 480-499 17 17 17 23 23 23 F F F
23592024-01-24T19:06:25.224Z 25 500-519 13 13 13 17 17 17 F F F
23602024-01-24T19:06:25.224Z 26 520-539 14 14 14 19 19 19 F F F
23612024-01-24T19:06:25.224Z 27 540-559 20 20 20 29 29 29 F F F
23622024-01-24T19:06:25.224Z 28 560-579 16 16 16 21 21 21 F F F
23632024-01-24T19:06:25.224Z 29 580-599 8 8 8 10 10 10 F F F
23642024-01-24T19:06:25.225ZMax gen: 21, Max flush: 30
23652024-01-24T19:06:25.225Z
23662024-01-24T19:06:25.225ZBegin loop 11
23672024-01-24T19:06:25.225ZDownstairs to restart: 0
23682024-01-24T19:06:25.251Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.882711478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217}
23692024-01-24T19:06:25.251Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883206418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217}
23702024-01-24T19:06:25.251Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883237102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217}
23712024-01-24T19:06:25.251Z{"msg":"Crucible 147f8c5c-376e-4c4b-8931-bc7ddbbd11aa has session id: a0f4b6c4-472e-4def-a748-d0a2eac075b9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883260485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23722024-01-24T19:06:25.252Z{"msg":"Upstairs opts: Upstairs UUID: 147f8c5c-376e-4c4b-8931-bc7ddbbd11aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883318675Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23732024-01-24T19:06:25.252Z{"msg":"Crucible stats registered with UUID: 147f8c5c-376e-4c4b-8931-bc7ddbbd11aa","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883359337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23742024-01-24T19:06:25.252ZCrucible runtime is spawned
23752024-01-24T19:06:25.252Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.8836165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217}
23762024-01-24T19:06:25.252Z{"msg":"147f8c5c-376e-4c4b-8931-bc7ddbbd11aa active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883884095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23772024-01-24T19:06:25.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.883952188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23782024-01-24T19:06:25.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884030477Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23792024-01-24T19:06:25.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884074241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23802024-01-24T19:06:25.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884154238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23812024-01-24T19:06:25.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.88423521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23822024-01-24T19:06:25.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884333603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23832024-01-24T19:06:25.253Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884380314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23842024-01-24T19:06:25.253Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884462665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23852024-01-24T19:06:25.253Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884547348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23862024-01-24T19:06:25.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884592787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23872024-01-24T19:06:25.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884765938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23882024-01-24T19:06:25.254Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.884812379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23892024-01-24T19:06:25.254Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886094994Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23902024-01-24T19:06:25.254Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886158383Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23912024-01-24T19:06:25.254Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886193791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23922024-01-24T19:06:25.255Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886249162Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23932024-01-24T19:06:25.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886441762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23942024-01-24T19:06:25.256Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.886516993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23952024-01-24T19:06:25.256Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.887193163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23962024-01-24T19:06:25.256Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.887270685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23972024-01-24T19:06:25.292Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.887718895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23982024-01-24T19:06:25.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.8880299Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
23992024-01-24T19:06:25.292Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888120026Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24002024-01-24T19:06:25.292Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888170395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24012024-01-24T19:06:25.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888213919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24022024-01-24T19:06:25.300Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888257835Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24032024-01-24T19:06:25.301Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888291431Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24042024-01-24T19:06:25.301Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888654989Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24052024-01-24T19:06:25.301Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888763623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24062024-01-24T19:06:25.301Z{"msg":"[0]R flush_numbers[0..12]: [25, 29, 26, 27, 25, 30, 30, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888811158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24072024-01-24T19:06:25.301Z{"msg":"[0]R generation[0..12]: [19, 20, 20, 20, 19, 21, 21, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888877957Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24082024-01-24T19:06:25.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.888925508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24092024-01-24T19:06:25.301ZWait for a query_work_queue command to finish before sending IO
24102024-01-24T19:06:25.301Z{"msg":"[1]R flush_numbers[0..12]: [25, 29, 26, 27, 25, 30, 30, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.88898231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24112024-01-24T19:06:25.301Z{"msg":"[1]R generation[0..12]: [19, 20, 20, 20, 19, 21, 21, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889079348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24122024-01-24T19:06:25.302Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889127173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24132024-01-24T19:06:25.302Z{"msg":"[2]R flush_numbers[0..12]: [25, 29, 26, 27, 25, 30, 30, 17, 20, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889193192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24142024-01-24T19:06:25.302Z{"msg":"[2]R generation[0..12]: [19, 20, 20, 20, 19, 21, 21, 13, 15, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889238176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24152024-01-24T19:06:25.302Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889295451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24162024-01-24T19:06:25.304Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889340523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24172024-01-24T19:06:25.304Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889410614Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24182024-01-24T19:06:25.304Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889438446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24192024-01-24T19:06:25.305Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889460451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24202024-01-24T19:06:25.305Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889512645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24212024-01-24T19:06:25.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889536925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24222024-01-24T19:06:25.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889558896Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24232024-01-24T19:06:25.305Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889581126Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24242024-01-24T19:06:25.305Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889603787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24252024-01-24T19:06:25.305Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889624606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24262024-01-24T19:06:25.305Z{"msg":"147f8c5c-376e-4c4b-8931-bc7ddbbd11aa is now active with session: a0f4b6c4-472e-4def-a748-d0a2eac075b9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889647857Z"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24272024-01-24T19:06:25.305Z,"hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24282024-01-24T19:06:25.305Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889673443Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217,"session_id":"a0f4b6c4-472e-4def-a748-d0a2eac075b9"}
24292024-01-24T19:06:25.305Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:23.889695421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1217}
24302024-01-24T19:06:25.305ZLoading write count information from file "/tmp/test_repair_verify.data"
24312024-01-24T19:06:25.305ZRun Repair workload
24322024-01-24T19:06:25.305Z01/30 Flush
24332024-01-24T19:06:25.305Z02/30 Write block 151 len 20480 data: 5 5 5 5 3
24342024-01-24T19:06:25.305Z03/30 Write block 465 len 20480 data: 2 2 2 2 2
24352024-01-24T19:06:25.305Z04/30 Write block 417 len 24576 data: 3 3 3 2 2 3
24362024-01-24T19:06:25.308Z05/30 Flush
24372024-01-24T19:06:25.308Z06/30 Write block 4 len 40960 data: 3 3 3 3 4 4 4 3 3 3
24382024-01-24T19:06:25.308Z07/30 Write block 189 len 40960 data: 2 2 2 2 2 2 2 2 2 2
24392024-01-24T19:06:25.308Z08/30 Read block 60 len 36864
24402024-01-24T19:06:25.308Z09/30 Write block 315 len 20480 data: 4 4 4 4 4
24412024-01-24T19:06:25.308Z10/30 Write block 588 len 32768 data: 2 2 2 2 3 3 3 3
24422024-01-24T19:06:25.308Z11/30 Write block 405 len 4096 data: 2
24432024-01-24T19:06:25.308Z12/30 Write block 29 len 24576 data: 5 5 6 6 6 5
24442024-01-24T19:06:25.308Z13/30 Write block 380 len 8192 data: 2 2
24452024-01-24T19:06:25.308Z14/30 Read block 148 len 24576
24462024-01-24T19:06:25.308Z15/30 Write block 591 len 36864 data: 3 4 4 4 4 3 3 3 2
24472024-01-24T19:06:25.308Z16/30 Write block 171 len 8192 data: 5 5
24482024-01-24T19:06:25.308Z17/30 Read block 76 len 28672
24492024-01-24T19:06:25.308Z18/30 Read block 119 len 24576
24502024-01-24T19:06:25.308Z19/30 Read block 264 len 20480
24512024-01-24T19:06:25.325Z20/30 Read block 105 len 36864
24522024-01-24T19:06:25.325Z21/30 Write block 465 len 32768 data: 3 3 3 3 3 2 2 4
24532024-01-24T19:06:25.326Z22/30 Write block 277 len 12288 data: 2 3 6
24542024-01-24T19:06:25.326Z23/30 Read block 233 len 16384
24552024-01-24T19:06:25.326Z24/30 Write block 216 len 28672 data: 3 3 3 3 3 3 3
24562024-01-24T19:06:25.326Z25/30 Write block 578 len 8192 data: 2 2
24572024-01-24T19:06:25.326Z26/30 Write block 325 len 8192 data: 4 4
24582024-01-24T19:06:25.326Z27/30 Write block 258 len 16384 data: 2 2 2 2
24592024-01-24T19:06:25.326Z28/30 Write block 453 len 36864 data: 3 3 3 3 3 5 5 5 4
24602024-01-24T19:06:25.326Z29/30 Write block 239 len 32768 data: 2 2 3 3 4 4 4 5
24612024-01-24T19:06:25.327Z30/30 Read block 49 len 4096
24622024-01-24T19:06:25.327Z----------------------------------------------------------------
24632024-01-24T19:06:25.327Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:25
24642024-01-24T19:06:25.327ZStates: Active Active Active
24652024-01-24T19:06:25.328ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24662024-01-24T19:06:25.328Z 6 Acked 1005 Write 10 Sent Done Done false
24672024-01-24T19:06:25.328Z 7 Acked 1006 Write 10 Sent Done Done false
24682024-01-24T19:06:25.328Z 8 Acked 1007 Read 9 Sent Done Done false
24692024-01-24T19:06:25.328Z 9 Acked 1008 Write 5 Sent Done Done false
24702024-01-24T19:06:25.328Z 10 Acked 1009 Write 8 Sent Done Done false
24712024-01-24T19:06:25.328Z 11 Acked 1010 Write 1 Sent Done Done false
24722024-01-24T19:06:25.328Z 12 Acked 1011 Write 6 Sent Done Done false
24732024-01-24T19:06:25.328Z 13 Acked 1012 Write 2 Sent Done Done false
24742024-01-24T19:06:25.328Z 14 Acked 1013 Read 6 Sent Done Done false
24752024-01-24T19:06:25.328Z 15 Acked 1014 Write 9 Sent Done Done false
24762024-01-24T19:06:25.328Z 16 Acked 1015 Write 2 Sent Done Done false
24772024-01-24T19:06:25.328Z 17 Acked 1016 Read 7 Sent Done Done false
24782024-01-24T19:06:25.328Z 18 Acked 1017 Read 6 Sent Done Done false
24792024-01-24T19:06:25.328Z 19 Acked 1018 Read 5 Sent Done Done false
24802024-01-24T19:06:25.328Z 20 Acked 1019 Read 9 Sent Done Done false
24812024-01-24T19:06:25.328Z 21 Acked 1020 Write 8 Sent Done Done false
24822024-01-24T19:06:25.328Z 22 Acked 1021 Write 3 Sent Done Done false
24832024-01-24T19:06:25.328Z 23 Acked 1022 Read 4 Sent Done Done false
24842024-01-24T19:06:25.329Z 24 Acked 1023 Write 7 Sent Done Done false
24852024-01-24T19:06:25.329Z 25 Acked 1024 Write 2 Sent Done Done false
24862024-01-24T19:06:25.329Z 26 Acked 1025 Write 2 Sent Done Done false
24872024-01-24T19:06:25.329Z 27 Acked 1026 Write 4 Sent Done Done false
24882024-01-24T19:06:25.329Z 28 Acked 1027 Write 9 Sent Done Done false
24892024-01-24T19:06:25.329Z 29 Acked 1028 Write 8 Sent Done Done false
24902024-01-24T19:06:25.329Z 30 Acked 1029 Read 1 Sent Done Done false
24912024-01-24T19:06:25.329Z STATES DS:0 DS:1 DS:2 TOTAL
24922024-01-24T19:06:25.329Z New 0 0 0 0
24932024-01-24T19:06:25.329Z Sent 25 0 0 25
24942024-01-24T19:06:25.329Z Done 0 25 25 50
24952024-01-24T19:06:25.330Z Skipped 0 0 0 0
24962024-01-24T19:06:25.330Z Error 0 0 0 0
24972024-01-24T19:06:25.330ZLast Flush: 1004 1004 1004
24982024-01-24T19:06:25.330ZDownstairs last five completed: 1004 1003 1002 1001 1000
24992024-01-24T19:06:25.330ZUpstairs last five completed: 30 29 28 27 26
25002024-01-24T19:06:25.330ZWrote out file "/tmp/test_repair_verify.data"
25012024-01-24T19:06:25.330Z
25022024-01-24T19:06:25.330ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25032024-01-24T19:06:25.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25042024-01-24T19:06:25.342Z 0 000-019 19 19 19 25 25 25 F T T <---
25052024-01-24T19:06:25.342Z 1 020-039 20 20 20 29 29 29 F T T <---
25062024-01-24T19:06:25.342Z 2 040-059 20 20 20 26 26 26 F F F
25072024-01-24T19:06:25.342Z 3 060-079 20 20 20 27 27 27 F F F
25082024-01-24T19:06:25.343Z 4 080-099 19 19 19 25 25 25 F F F
25092024-01-24T19:06:25.343Z 5 100-119 21 21 21 30 30 30 F F F
25102024-01-24T19:06:25.343Z 6 120-139 21 21 21 30 30 30 F F F
25112024-01-24T19:06:25.344Z 7 140-159 22 22 22 32 32 32 F F F
25122024-01-24T19:06:25.344Z 8 160-179 15 15 15 20 20 20 F T T <---
25132024-01-24T19:06:25.344Z 9 180-199 18 18 18 24 24 24 F T T <---
25142024-01-24T19:06:25.344Z 10 200-219 18 18 18 24 24 24 F T T <---
25152024-01-24T19:06:25.344Z 11 220-239 18 18 18 24 24 24 F T T <---
25162024-01-24T19:06:25.344Z 12 240-259 14 14 14 18 18 18 F T T <---
25172024-01-24T19:06:25.344Z 13 260-279 20 20 20 28 28 28 F T T <---
25182024-01-24T19:06:25.344Z 14 280-299 20 20 20 28 28 28 F F F
25192024-01-24T19:06:25.344Z 15 300-319 18 18 18 24 24 24 F T T <---
25202024-01-24T19:06:25.344Z 16 320-339 18 18 18 24 24 24 F T T <---
25212024-01-24T19:06:25.344Z 17 340-359 10 10 10 13 13 13 F F F
25222024-01-24T19:06:25.344Z 18 360-379 18 18 18 24 24 24 F F F
25232024-01-24T19:06:25.344Z 19 380-399 16 16 16 21 21 21 F T T <---
25242024-01-24T19:06:25.344Z 20 400-419 22 22 22 32 32 32 F T T <---
25252024-01-24T19:06:25.345Z 21 420-439 22 22 22 32 32 32 F F F
25262024-01-24T19:06:25.345Z 22 440-459 20 20 20 28 28 28 F T T <---
25272024-01-24T19:06:25.345Z 23 460-479 22 22 22 32 32 32 F T T <---
25282024-01-24T19:06:25.345Z 24 480-499 17 17 17 23 23 23 F F F
25292024-01-24T19:06:25.345Z 25 500-519 13 13 13 17 17 17 F F F
25302024-01-24T19:06:25.345Z 26 520-539 14 14 14 19 19 19 F F F
25312024-01-24T19:06:25.345Z 27 540-559 20 20 20 29 29 29 F F F
25322024-01-24T19:06:25.345Z 28 560-579 16 16 16 21 21 21 F T T <---
25332024-01-24T19:06:25.354Z 29 580-599 8 8 8 10 10 10 F T T <---
25342024-01-24T19:06:25.354ZMax gen: 22, Max flush: 32
25352024-01-24T19:06:25.354ZError: Difference in extent metadata found!
25362024-01-24T19:06:25.354ZA Difference in extent metadata is expected here
25372024-01-24T19:06:25.354ZOn loop 11
25382024-01-24T19:06:25.354Z
25392024-01-24T19:06:25.354ZVerifying data now
25402024-01-24T19:06:30.735ZNo change after verify
25412024-01-24T19:06:30.736ZLoop: 11 Downstairs dump after verify (and repair):
25422024-01-24T19:06:30.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25432024-01-24T19:06:30.759Z 0 000-019 23 23 23 33 33 33 F F F
25442024-01-24T19:06:30.759Z 1 020-039 23 23 23 33 33 33 F F F
25452024-01-24T19:06:30.759Z 2 040-059 20 20 20 26 26 26 F F F
25462024-01-24T19:06:30.759Z 3 060-079 20 20 20 27 27 27 F F F
25472024-01-24T19:06:30.759Z 4 080-099 19 19 19 25 25 25 F F F
25482024-01-24T19:06:30.760Z 5 100-119 21 21 21 30 30 30 F F F
25492024-01-24T19:06:30.782Z 6 120-139 21 21 21 30 30 30 F F F
25502024-01-24T19:06:30.782Z 7 140-159 22 22 22 32 32 32 F F F
25512024-01-24T19:06:30.782Z 8 160-179 23 23 23 33 33 33 F F F
25522024-01-24T19:06:30.783Z 9 180-199 23 23 23 33 33 33 F F F
25532024-01-24T19:06:30.783Z 10 200-219 23 23 23 33 33 33 F F F
25542024-01-24T19:06:30.783Z 11 220-239 23 23 23 33 33 33 F F F
25552024-01-24T19:06:30.783Z 12 240-259 23 23 23 33 33 33 F F F
25562024-01-24T19:06:30.784Z 13 260-279 23 23 23 33 33 33 F F F
25572024-01-24T19:06:30.784Z 14 280-299 20 20 20 28 28 28 F F F
25582024-01-24T19:06:30.784Z 15 300-319 23 23 23 33 33 33 F F F
25592024-01-24T19:06:30.784Z 16 320-339 23 23 23 33 33 33 F F F
25602024-01-24T19:06:30.785Z 17 340-359 10 10 10 13 13 13 F F F
25612024-01-24T19:06:30.785Z 18 360-379 18 18 18 24 24 24 F F F
25622024-01-24T19:06:30.785Z 19 380-399 23 23 23 33 33 33 F F F
25632024-01-24T19:06:30.785Z 20 400-419 23 23 23 33 33 33 F F F
25642024-01-24T19:06:30.786Z 21 420-439 22 22 22 32 32 32 F F F
25652024-01-24T19:06:30.786Z 22 440-459 23 23 23 33 33 33 F F F
25662024-01-24T19:06:30.786Z 23 460-479 23 23 23 33 33 33 F F F
25672024-01-24T19:06:30.786Z 24 480-499 17 17 17 23 23 23 F F F
25682024-01-24T19:06:30.787Z 25 500-519 13 13 13 17 17 17 F F F
25692024-01-24T19:06:30.787Z 26 520-539 14 14 14 19 19 19 F F F
25702024-01-24T19:06:30.787Z 27 540-559 20 20 20 29 29 29 F F F
25712024-01-24T19:06:30.787Z 28 560-579 23 23 23 33 33 33 F F F
25722024-01-24T19:06:30.787Z 29 580-599 23 23 23 33 33 33 F F F
25732024-01-24T19:06:30.787ZMax gen: 23, Max flush: 33
25742024-01-24T19:06:30.798Z
25752024-01-24T19:06:30.798ZBegin loop 12
25762024-01-24T19:06:30.799ZDownstairs to restart: 1
25772024-01-24T19:06:30.808Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.448488373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225}
25782024-01-24T19:06:30.808Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.448984714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225}
25792024-01-24T19:06:30.808Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449055083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225}
25802024-01-24T19:06:30.809Z{"msg":"Crucible bc47c518-3e18-455c-9738-3f17d53034b0 has session id: 7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449124144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25812024-01-24T19:06:30.809Z{"msg":"Upstairs opts: Upstairs UUID: bc47c518-3e18-455c-9738-3f17d53034b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449182226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25822024-01-24T19:06:30.809ZCrucible runtime is spawned
25832024-01-24T19:06:30.809Z{"msg":"Crucible stats registered with UUID: bc47c518-3e18-455c-9738-3f17d53034b0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449351869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25842024-01-24T19:06:30.810Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449436922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225}
25852024-01-24T19:06:30.810Z{"msg":"bc47c518-3e18-455c-9738-3f17d53034b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449672087Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25862024-01-24T19:06:30.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449730919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25872024-01-24T19:06:30.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449758635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25882024-01-24T19:06:30.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449794817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25892024-01-24T19:06:30.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.449865728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25902024-01-24T19:06:30.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.44991213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25912024-01-24T19:06:30.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450146872Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25922024-01-24T19:06:30.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450312138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25932024-01-24T19:06:30.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450400717Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25942024-01-24T19:06:30.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450501401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25952024-01-24T19:06:30.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450538259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25962024-01-24T19:06:30.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450561774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25972024-01-24T19:06:30.813Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.450612071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25982024-01-24T19:06:30.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451566024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
25992024-01-24T19:06:30.813Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451656033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26002024-01-24T19:06:30.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451820981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26012024-01-24T19:06:30.813Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451885518Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26022024-01-24T19:06:30.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451915985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26032024-01-24T19:06:30.814Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.451957288Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26042024-01-24T19:06:30.814Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.452601586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26052024-01-24T19:06:30.814Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.45292552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26062024-01-24T19:06:30.814Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453401233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26072024-01-24T19:06:30.847Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453489719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26082024-01-24T19:06:30.847Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453572641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26092024-01-24T19:06:30.847Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453653881Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26102024-01-24T19:06:30.847Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453878185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26112024-01-24T19:06:30.848Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453941748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26122024-01-24T19:06:30.848Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.453968702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26132024-01-24T19:06:30.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454394334Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26142024-01-24T19:06:30.853Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454522533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26152024-01-24T19:06:30.853Z{"msg":"[0]R flush_numbers[0..12]: [33, 33, 26, 27, 25, 30, 30, 32, 33, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454609783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26162024-01-24T19:06:30.853Z{"msg":"[0]R generation[0..12]: [23, 23, 20, 20, 19, 21, 21, 22, 23, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454684408Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26172024-01-24T19:06:30.854Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454757571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"Wait for a query_work_queue command to finish before sending IO
26182024-01-24T19:06:30.854Z":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26192024-01-24T19:06:30.854Z{"msg":"[1]R flush_numbers[0..12]: [33, 33, 26, 27, 25, 30, 30, 32, 33, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454910449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26202024-01-24T19:06:30.854Z{"msg":"[1]R generation[0..12]: [23, 23, 20, 20, 19, 21, 21, 22, 23, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.454988567Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26212024-01-24T19:06:30.854Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455143102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26222024-01-24T19:06:30.854Z{"msg":"[2]R flush_numbers[0..12]: [33, 33, 26, 27, 25, 30, 30, 32, 33, 33, 33, 33]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455220842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26232024-01-24T19:06:30.854Z{"msg":"[2]R generation[0..12]: [23, 23, 20, 20, 19, 21, 21, 22, 23, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455268229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26242024-01-24T19:06:30.854Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455347348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26252024-01-24T19:06:30.854Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455425859Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26262024-01-24T19:06:30.855Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455501911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26272024-01-24T19:06:30.856Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455577617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26282024-01-24T19:06:30.856ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26292024-01-24T19:06:30.856Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455652384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26302024-01-24T19:06:30.856Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455724643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26312024-01-24T19:06:30.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455751408Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26322024-01-24T19:06:30.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455795543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26332024-01-24T19:06:30.856Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455820985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26342024-01-24T19:06:30.856Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455864583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26352024-01-24T19:06:30.856Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455888444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26362024-01-24T19:06:30.856Z{"msg":"bc47c518-3e18-455c-9738-3f17d53034b0 is now active with session: 7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455930369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26372024-01-24T19:06:30.856Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.455953705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225,"session_id":"7f3ee4f5-35b9-4fe7-92e8-1bba4a3b7874"}
26382024-01-24T19:06:30.857Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:29.456011386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1225}
26392024-01-24T19:06:30.857ZLoading write count information from file "/tmp/test_repair_verify.data"
26402024-01-24T19:06:30.857ZRun Repair workload
26412024-01-24T19:06:30.857Z01/30 Write block 26 len 16384 data: 5 5 7 6
26422024-01-24T19:06:30.857Z02/30 Read block 136 len 20480
26432024-01-24T19:06:30.858Z03/30 Read block 254 len 28672
26442024-01-24T19:06:30.872Z04/30 Flush
26452024-01-24T19:06:30.872Z05/30 Flush
26462024-01-24T19:06:30.872Z06/30 Write block 185 len 16384 data: 4 4 3 3
26472024-01-24T19:06:30.872Z07/30 Read block 116 len 8192
26482024-01-24T19:06:30.872Z08/30 Read block 497 len 40960
26492024-01-24T19:06:30.872Z09/30 Flush
26502024-01-24T19:06:30.872Z10/30 Write block 536 len 40960 data: 4 4 3 3 2 3 4 4 4 3
26512024-01-24T19:06:30.872Z11/30 Read block 30 len 24576
26522024-01-24T19:06:30.872Z12/30 Write block 180 len 12288 data: 2 3 3
26532024-01-24T19:06:30.872Z13/30 Write block 282 len 8192 data: 7 6
26542024-01-24T19:06:30.873Z14/30 Write block 481 len 36864 data: 3 4 3 3 3 3 3 3 3
26552024-01-24T19:06:30.873Z15/30 Read block 139 len 12288
26562024-01-24T19:06:30.873Z16/30 Read block 483 len 20480
26572024-01-24T19:06:30.873Z17/30 Write block 180 len 4096 data: 3
26582024-01-24T19:06:30.873Z18/30 Read block 95 len 28672
26592024-01-24T19:06:30.873Z19/30 Write block 214 len 20480 data: 3 3 4 4 4
26602024-01-24T19:06:30.873Z20/30 Write block 580 len 40960 data: 2 2 2 2 2 2 2 2 3 3
26612024-01-24T19:06:30.873Z21/30 Write block 525 len 16384 data: 4 4 3 3
26622024-01-24T19:06:30.873Z22/30 Write block 58 len 4096 data: 3
26632024-01-24T19:06:30.873Z23/30 Write block 222 len 8192 data: 4 2
26642024-01-24T19:06:30.873Z24/30 Write block 63 len 12288 data: 2 3 3
26652024-01-24T19:06:30.873Z25/30 Read block 9 len 8192
26662024-01-24T19:06:30.873Z26/30 Read block 102 len 32768
26672024-01-24T19:06:30.873Z27/30 Read block 475 len 40960
26682024-01-24T19:06:30.873Z28/30 Read block 500 len 40960
26692024-01-24T19:06:30.873Z29/30 Read block 400 len 16384
26702024-01-24T19:06:30.873Z30/30 Write block 400 len 36864 data: 3 3 3 2 2 3 3 3 3
26712024-01-24T19:06:30.873Z----------------------------------------------------------------
26722024-01-24T19:06:30.873Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26732024-01-24T19:06:30.873ZStates: Active Active Active
26742024-01-24T19:06:30.873ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26752024-01-24T19:06:30.873Z 1 Acked 1000 Write 4 Done Done Done false
26762024-01-24T19:06:30.873Z 2 Acked 1001 Read 5 Done Sent Done false
26772024-01-24T19:06:30.874Z 3 Acked 1002 Read 7 Done Sent Done false
26782024-01-24T19:06:30.874Z 4 Acked 1003 Flush 0 Done Sent Done false
26792024-01-24T19:06:30.874Z 5 Acked 1004 Flush 0 Done Sent Done false
26802024-01-24T19:06:30.874Z 6 Acked 1005 Write 4 Done Sent Done false
26812024-01-24T19:06:30.874Z 7 Acked 1006 Read 2 Done Sent Done false
26822024-01-24T19:06:30.874Z 8 Acked 1007 Read 10 Done Sent Done false
26832024-01-24T19:06:30.874Z 9 Acked 1008 Flush 0 Done Sent Done false
26842024-01-24T19:06:30.875Z 10 Acked 1009 Write 10 Done Sent Done false
26852024-01-24T19:06:30.875Z 11 Acked 1010 Read 6 Done Sent Done false
26862024-01-24T19:06:30.875Z 12 Acked 1011 Write 3 Done Sent Done false
26872024-01-24T19:06:30.875Z 13 Acked 1012 Write 2 Done Sent Done false
26882024-01-24T19:06:30.875Z 14 Acked 1013 Write 9 Done Sent Done false
26892024-01-24T19:06:30.875Z 15 Acked 1014 Read 3 Done Sent Done false
26902024-01-24T19:06:30.875Z 16 Acked 1015 Read 5 Done Sent Done false
26912024-01-24T19:06:30.875Z 17 Acked 1016 Write 1 Done Sent Done false
26922024-01-24T19:06:30.875Z 18 Acked 1017 Read 7 Done Sent Done false
26932024-01-24T19:06:30.875Z 19 Acked 1018 Write 5 Done Sent Done false
26942024-01-24T19:06:30.875Z 20 Acked 1019 Write 10 Done Sent Done false
26952024-01-24T19:06:30.875Z 21 Acked 1020 Write 4 Done Sent Done false
26962024-01-24T19:06:30.875Z 22 Acked 1021 Write 1 Done Sent Done false
26972024-01-24T19:06:30.875Z 23 Acked 1022 Write 2 Done Sent Done false
26982024-01-24T19:06:30.875Z 24 Acked 1023 Write 3 Done Sent Done false
26992024-01-24T19:06:30.875Z 25 Acked 1024 Read 2 Done Sent Done false
27002024-01-24T19:06:30.875Z 26 Acked 1025 Read 8 Done Sent Done false
27012024-01-24T19:06:30.875Z 27 Acked 1026 Read 10 Done Sent Done false
27022024-01-24T19:06:30.875Z 28 Acked 1027 Read 10 Done Sent Done false
27032024-01-24T19:06:30.875Z 29 Acked 1028 Read 4 Done Sent Done false
27042024-01-24T19:06:30.875Z 30 Acked 1029 Write 9 Sent Sent Sent false
27052024-01-24T19:06:30.875Z STATES DS:0 DS:1 DS:2 TOTAL
27062024-01-24T19:06:30.875Z New 0 0 0 0
27072024-01-24T19:06:30.876Z Sent 1 29 1 31
27082024-01-24T19:06:30.876Z Done 29 1 29 59
27092024-01-24T19:06:30.876Z Skipped 0 0 0 0
27102024-01-24T19:06:30.876Z Error 0 0 0 0
27112024-01-24T19:06:30.876ZLast Flush: 1008 0 1008
27122024-01-24T19:06:30.876ZDownstairs last five completed:
27132024-01-24T19:06:30.876ZUpstairs last five completed: 30 29 28 27 26
27142024-01-24T19:06:30.914ZWrote out file "/tmp/test_repair_verify.data"
27152024-01-24T19:06:30.914Z
27162024-01-24T19:06:30.914ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27172024-01-24T19:06:30.919ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27182024-01-24T19:06:30.919Z 0 000-019 23 23 23 33 33 33 F F F
27192024-01-24T19:06:30.919Z 1 020-039 24 23 24 34 33 34 F T F <---
27202024-01-24T19:06:30.919Z 2 040-059 20 20 20 26 26 26 T F T <---
27212024-01-24T19:06:30.919Z 3 060-079 20 20 20 27 27 27 T F T <---
27222024-01-24T19:06:30.919Z 4 080-099 19 19 19 25 25 25 F F F
27232024-01-24T19:06:30.919Z 5 100-119 21 21 21 30 30 30 F F F
27242024-01-24T19:06:30.919Z 6 120-139 21 21 21 30 30 30 F F F
27252024-01-24T19:06:30.919Z 7 140-159 22 22 22 32 32 32 F F F
27262024-01-24T19:06:30.919Z 8 160-179 23 23 23 33 33 33 F F F
27272024-01-24T19:06:30.919Z 9 180-199 24 23 24 36 33 36 T F T <---
27282024-01-24T19:06:30.920Z 10 200-219 23 23 23 33 33 33 T F T <---
27292024-01-24T19:06:30.920Z 11 220-239 23 23 23 33 33 33 T F T <---
27302024-01-24T19:06:30.920Z 12 240-259 23 23 23 33 33 33 F F F
27312024-01-24T19:06:30.920Z 13 260-279 23 23 23 33 33 33 F F F
27322024-01-24T19:06:30.920Z 14 280-299 20 20 20 28 28 28 T F T <---
27332024-01-24T19:06:30.920Z 15 300-319 23 23 23 33 33 33 F F F
27342024-01-24T19:06:30.920Z 16 320-339 23 23 23 33 33 33 F F F
27352024-01-24T19:06:30.920Z 17 340-359 10 10 10 13 13 13 F F F
27362024-01-24T19:06:30.920Z 18 360-379 18 18 18 24 24 24 F F F
27372024-01-24T19:06:30.920Z 19 380-399 23 23 23 33 33 33 F F F
27382024-01-24T19:06:30.920Z 20 400-419 23 23 23 33 33 33 T F T <---
27392024-01-24T19:06:30.921Z 21 420-439 22 22 22 32 32 32 F F F
27402024-01-24T19:06:30.921Z 22 440-459 23 23 23 33 33 33 F F F
27412024-01-24T19:06:30.921Z 23 460-479 23 23 23 33 33 33 F F F
27422024-01-24T19:06:30.921Z 24 480-499 17 17 17 23 23 23 T F T <---
27432024-01-24T19:06:30.921Z 25 500-519 13 13 13 17 17 17 F F F
27442024-01-24T19:06:30.921Z 26 520-539 14 14 14 19 19 19 T F T <---
27452024-01-24T19:06:30.921Z 27 540-559 20 20 20 29 29 29 T F T <---
27462024-01-24T19:06:30.921Z 28 560-579 23 23 23 33 33 33 F F F
27472024-01-24T19:06:30.921Z 29 580-599 23 23 23 33 33 33 T F T <---
27482024-01-24T19:06:30.921ZMax gen: 24, Max flush: 36
27492024-01-24T19:06:30.921ZError: Difference in extent metadata found!
27502024-01-24T19:06:30.921ZA Difference in extent metadata is expected here
27512024-01-24T19:06:30.921ZOn loop 12
27522024-01-24T19:06:30.921Z
27532024-01-24T19:06:30.926ZVerifying data now
27542024-01-24T19:06:36.041ZNo change after verify
27552024-01-24T19:06:36.041ZLoop: 12 Downstairs dump after verify (and repair):
27562024-01-24T19:06:36.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27572024-01-24T19:06:36.472Z 0 000-019 23 23 23 33 33 33 F F F
27582024-01-24T19:06:36.472Z 1 020-039 24 24 24 34 34 34 F F F
27592024-01-24T19:06:36.472Z 2 040-059 25 25 25 37 37 37 F F F
27602024-01-24T19:06:36.472Z 3 060-079 25 25 25 37 37 37 F F F
27612024-01-24T19:06:36.472Z 4 080-099 19 19 19 25 25 25 F F F
27622024-01-24T19:06:36.472Z 5 100-119 21 21 21 30 30 30 F F F
27632024-01-24T19:06:36.472Z 6 120-139 21 21 21 30 30 30 F F F
27642024-01-24T19:06:36.473Z 7 140-159 22 22 22 32 32 32 F F F
27652024-01-24T19:06:36.473Z 8 160-179 23 23 23 33 33 33 F F F
27662024-01-24T19:06:36.473Z 9 180-199 25 25 25 37 37 37 F F F
27672024-01-24T19:06:36.473Z 10 200-219 25 25 25 37 37 37 F F F
27682024-01-24T19:06:36.473Z 11 220-239 25 25 25 37 37 37 F F F
27692024-01-24T19:06:36.473Z 12 240-259 23 23 23 33 33 33 F F F
27702024-01-24T19:06:36.473Z 13 260-279 23 23 23 33 33 33 F F F
27712024-01-24T19:06:36.473Z 14 280-299 25 25 25 37 37 37 F F F
27722024-01-24T19:06:36.474Z 15 300-319 23 23 23 33 33 33 F F F
27732024-01-24T19:06:36.474Z 16 320-339 23 23 23 33 33 33 F F F
27742024-01-24T19:06:36.474Z 17 340-359 10 10 10 13 13 13 F F F
27752024-01-24T19:06:36.474Z 18 360-379 18 18 18 24 24 24 F F F
27762024-01-24T19:06:36.474Z 19 380-399 23 23 23 33 33 33 F F F
27772024-01-24T19:06:36.475Z 20 400-419 25 25 25 37 37 37 F F F
27782024-01-24T19:06:36.475Z 21 420-439 22 22 22 32 32 32 F F F
27792024-01-24T19:06:36.475Z 22 440-459 23 23 23 33 33 33 F F F
27802024-01-24T19:06:36.523Z 23 460-479 23 23 23 33 33 33 F F F
27812024-01-24T19:06:36.523Z 24 480-499 25 25 25 37 37 37 F F F
27822024-01-24T19:06:36.523Z 25 500-519 13 13 13 17 17 17 F F F
27832024-01-24T19:06:36.523Z 26 520-539 25 25 25 37 37 37 F F F
27842024-01-24T19:06:36.523Z 27 540-559 25 25 25 37 37 37 F F F
27852024-01-24T19:06:36.523Z 28 560-579 23 23 23 33 33 33 F F F
27862024-01-24T19:06:36.523Z 29 580-599 25 25 25 37 37 37 F F F
27872024-01-24T19:06:36.524ZMax gen: 25, Max flush: 37
27882024-01-24T19:06:36.524Z
27892024-01-24T19:06:36.524ZBegin loop 13
27902024-01-24T19:06:36.524ZDownstairs to restart: 2
27912024-01-24T19:06:37.162Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.714939675Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233}
27922024-01-24T19:06:37.163Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.715451981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233}
27932024-01-24T19:06:37.163Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.715520452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233}
27942024-01-24T19:06:37.163Z{"msg":"Crucible d9f36d6f-3545-4702-8551-a31e086f1bc8 has session id: c6f4ca2a-93e8-4a72-8338-015deb4f93ef","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.715552349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
27952024-01-24T19:06:37.163Z{"msg":"Upstairs opts: Upstairs UUID: d9f36d6f-3545-4702-8551-a31e086f1bc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
27962024-01-24T19:06:37.163Z,"time":"2024-01-24T19:06:34.715661576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
27972024-01-24T19:06:37.163Z{"msg":"Crucible stats registered with UUID: d9f36d6f-3545-4702-8551-a31e086f1bc8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.715719738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
27982024-01-24T19:06:37.163Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.71580712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233}
27992024-01-24T19:06:37.164Z{"msg":"d9f36d6f-3545-4702-8551-a31e086f1bc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716082021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28002024-01-24T19:06:37.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716151842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28012024-01-24T19:06:37.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716198674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28022024-01-24T19:06:37.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716321127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28032024-01-24T19:06:37.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716367617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28042024-01-24T19:06:37.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.71640552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28052024-01-24T19:06:37.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716432685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28062024-01-24T19:06:37.164Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716636486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28072024-01-24T19:06:37.164Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716816194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28082024-01-24T19:06:37.165Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716939076Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28092024-01-24T19:06:37.165Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.716986576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28102024-01-24T19:06:37.165Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.717088261Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28112024-01-24T19:06:37.165Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.717131098Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28122024-01-24T19:06:37.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718298967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28132024-01-24T19:06:37.168Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718377491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28142024-01-24T19:06:37.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718453809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28152024-01-24T19:06:37.168Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718580862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28162024-01-24T19:06:37.168Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718654668Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28172024-01-24T19:06:37.168Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.718700257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28182024-01-24T19:06:37.168Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.71944656Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28192024-01-24T19:06:37.168Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.719523207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28202024-01-24T19:06:37.169Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.719717414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28212024-01-24T19:06:37.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720429401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28222024-01-24T19:06:37.171Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720494171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28232024-01-24T19:06:37.171Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720522068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28242024-01-24T19:06:37.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720667379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28252024-01-24T19:06:37.171Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720783294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28262024-01-24T19:06:37.171Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720843615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28272024-01-24T19:06:37.171Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720869463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28282024-01-24T19:06:37.171Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720893664Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28292024-01-24T19:06:37.174Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 37, 37, 25, 30, 30, 32, 33, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720937728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28302024-01-24T19:06:37.174Z{"msg":"[0]R generation[0..12]: [23, 24, 25, 25, 19, 21, 21, 22, 23, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720964597Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28312024-01-24T19:06:37.174Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.720989341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28322024-01-24T19:06:37.174Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 37, 37, 25, 30, 30, 32, 33, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721033748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28332024-01-24T19:06:37.175Z{"msg":"[1]R generation[0..12]: [23, 24, 25, 25, 19, 21, 21, 22, 23, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.72105995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28342024-01-24T19:06:37.178Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721083956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28352024-01-24T19:06:37.178Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 37, 37, 25, 30, 30, 32, 33, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721107392Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28362024-01-24T19:06:37.178Z{"msg":"[2]R generation[0..12]: [23, 24, 25, 25, 19, 21, 21, 22, 23, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721130041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28372024-01-24T19:06:37.178Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v"Wait for a query_work_queue command to finish before sending IO
28382024-01-24T19:06:37.178Z:0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721179174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28392024-01-24T19:06:37.179Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721204922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28402024-01-24T19:06:37.179Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721226981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28412024-01-24T19:06:37.179Z{"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721248304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28422024-01-24T19:06:37.183Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721270157Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28432024-01-24T19:06:37.184Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721292529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28442024-01-24T19:06:37.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721332659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28452024-01-24T19:06:37.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721357622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28462024-01-24T19:06:37.184Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721379689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28472024-01-24T19:06:37.185Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721422772Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28482024-01-24T19:06:37.185Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721446725Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28492024-01-24T19:06:37.186Z{"msg":"d9f36d6f-3545-4702-8551-a31e086f1bc8 is now active with session: c6f4ca2a-93e8-4a72-8338-015deb4f93ef","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721468607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28502024-01-24T19:06:37.186Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.72150877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233,"session_id":"c6f4ca2a-93e8-4a72-8338-015deb4f93ef"}
28512024-01-24T19:06:37.186Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:34.721532437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1233}
28522024-01-24T19:06:37.186ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28532024-01-24T19:06:37.186ZLoading write count information from file "/tmp/test_repair_verify.data"
28542024-01-24T19:06:37.186ZRun Repair workload
28552024-01-24T19:06:37.187Z01/30 Write block 504 len 8192 data: 3 3
28562024-01-24T19:06:37.187Z02/30 Read block 72 len 12288
28572024-01-24T19:06:37.187Z03/30 Write block 592 len 8192 data: 5 5
28582024-01-24T19:06:37.187Z04/30 Flush
28592024-01-24T19:06:37.187Z05/30 Write block 107 len 28672 data: 3 4 4 3 3 3 3
28602024-01-24T19:06:37.188Z06/30 Read block 372 len 16384
28612024-01-24T19:06:37.188Z07/30 Flush
28622024-01-24T19:06:37.188Z08/30 Write block 358 len 36864 data: 3 3 3 3 3 3 3 2 2
28632024-01-24T19:06:37.188Z09/30 Read block 347 len 12288
28642024-01-24T19:06:37.188Z10/30 Write block 475 len 32768 data: 2 2 2 2 2 3 4 5
28652024-01-24T19:06:37.188Z11/30 Read block 299 len 36864
28662024-01-24T19:06:37.188Z12/30 Write block 100 len 28672 data: 4 2 2 2 2 2 3
28672024-01-24T19:06:37.188Z13/30 Read block 356 len 28672
28682024-01-24T19:06:37.188Z14/30 Write block 56 len 24576 data: 3 3 4 4 3 2
28692024-01-24T19:06:37.188Z15/30 Write block 259 len 8192 data: 3 3
28702024-01-24T19:06:37.188Z16/30 Write block 294 len 12288 data: 5 3 2
28712024-01-24T19:06:37.188Z17/30 Read block 564 len 4096
28722024-01-24T19:06:37.189Z18/30 Flush
28732024-01-24T19:06:37.189Z19/30 Write block 222 len 4096 data: 5
28742024-01-24T19:06:37.189Z20/30 Flush
28752024-01-24T19:06:37.189Z21/30 Write block 518 len 16384 data: 4 4 4 4
28762024-01-24T19:06:37.189Z22/30 Write block 224 len 36864 data: 3 4 4 5 4 3 3 3 3
28772024-01-24T19:06:37.189Z23/30 Read block 475 len 32768
28782024-01-24T19:06:37.189Z24/30 Flush
28792024-01-24T19:06:37.191Z25/30 Write block 378 len 28672 data: 3 3 3 3 2 3 3
28802024-01-24T19:06:37.191Z26/30 Read block 196 len 28672
28812024-01-24T19:06:37.191Z27/30 Write block 115 len 4096 data: 2
28822024-01-24T19:06:37.191Z28/30 Read block 184 len 16384
28832024-01-24T19:06:37.191Z29/30 Write block 258 len 4096 data: 3
28842024-01-24T19:06:37.191Z30/30 Write block 373 len 12288 data: 2 2 2
28852024-01-24T19:06:37.191Z----------------------------------------------------------------
28862024-01-24T19:06:37.191Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:26
28872024-01-24T19:06:37.192ZStates: Active Active Active
28882024-01-24T19:06:37.192ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
28892024-01-24T19:06:37.192Z 5 Acked 1004 Write 7 Done Done Done false
28902024-01-24T19:06:37.192Z 6 Acked 1005 Read 4 Done Done Done false
28912024-01-24T19:06:37.273Z 7 Acked 1006 Flush 0 Done Done Sent false
28922024-01-24T19:06:37.274Z 8 Acked 1007 Write 9 Done Done Sent false
28932024-01-24T19:06:37.274Z 9 Acked 1008 Read 3 Done Done Sent false
28942024-01-24T19:06:37.274Z 10 Acked 1009 Write 8 Done Done Sent false
28952024-01-24T19:06:37.275Z 11 Acked 1010 Read 9 Done Done Sent false
28962024-01-24T19:06:37.275Z 12 Acked 1011 Write 7 Done Done Sent false
28972024-01-24T19:06:37.275Z 13 Acked 1012 Read 7 Done Done Sent false
28982024-01-24T19:06:37.275Z 14 Acked 1013 Write 6 Done Done Sent false
28992024-01-24T19:06:37.275Z 15 Acked 1014 Write 2 Done Done Sent false
29002024-01-24T19:06:37.275Z 16 Acked 1015 Write 3 Done Done Sent false
29012024-01-24T19:06:37.275Z 17 Acked 1016 Read 1 Done Done Sent false
29022024-01-24T19:06:37.275Z 18 Acked 1017 Flush 0 Done Done Sent false
29032024-01-24T19:06:37.275Z 19 Acked 1018 Write 1 Done Done Sent false
29042024-01-24T19:06:37.276Z 20 Acked 1019 Flush 0 Done Done Sent false
29052024-01-24T19:06:37.276Z 21 Acked 1020 Write 4 Done Done Sent false
29062024-01-24T19:06:37.276Z 22 Acked 1021 Write 9 Done Done Sent false
29072024-01-24T19:06:37.276Z 23 Acked 1022 Read 8 Done Done Sent false
29082024-01-24T19:06:37.276Z 24 Acked 1023 Flush 0 Done Done Sent false
29092024-01-24T19:06:37.277Z 25 Acked 1024 Write 7 Done Done Sent false
29102024-01-24T19:06:37.277Z 26 Acked 1025 Read 7 Done Done Sent false
29112024-01-24T19:06:37.277Z 27 Acked 1026 Write 1 Done Done Sent false
29122024-01-24T19:06:37.277Z 28 Acked 1027 Read 4 Done Done Sent false
29132024-01-24T19:06:37.277Z 29 Acked 1028 Write 1 Done Done Sent false
29142024-01-24T19:06:37.277Z 30 Acked 1029 Write 3 Sent Sent Sent false
29152024-01-24T19:06:37.279Z STATES DS:0 DS:1 DS:2 TOTAL
29162024-01-24T19:06:37.280Z New 0 0 0 0
29172024-01-24T19:06:37.281Z Sent 1 1 24 26
29182024-01-24T19:06:37.281Z Done 25 25 2 52
29192024-01-24T19:06:37.281Z Skipped 0 0 0 0
29202024-01-24T19:06:37.281Z Error 0 0 0 0
29212024-01-24T19:06:37.281ZLast Flush: 1023 1023 1003
29222024-01-24T19:06:37.281ZDownstairs last five completed: 1003 1002 1001 1000
29232024-01-24T19:06:37.284ZUpstairs last five completed: 30 29 28 27 26
29242024-01-24T19:06:37.284ZWrote out file "/tmp/test_repair_verify.data"
29252024-01-24T19:06:37.284Z
29262024-01-24T19:06:37.284ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29272024-01-24T19:06:37.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29282024-01-24T19:06:37.285Z 0 000-019 23 23 23 33 33 33 F F F
29292024-01-24T19:06:37.285Z 1 020-039 24 24 24 34 34 34 F F F
29302024-01-24T19:06:37.285Z 2 040-059 26 26 25 40 40 37 F F F <---
29312024-01-24T19:06:37.285Z 3 060-079 26 26 25 40 40 37 F F F <---
29322024-01-24T19:06:37.285Z 4 080-099 19 19 19 25 25 25 F F F
29332024-01-24T19:06:37.285Z 5 100-119 26 26 21 40 40 30 T T T <---
29342024-01-24T19:06:37.285Z 6 120-139 21 21 21 30 30 30 F F F
29352024-01-24T19:06:37.286Z 7 140-159 22 22 22 32 32 32 F F F
29362024-01-24T19:06:37.286Z 8 160-179 23 23 23 33 33 33 F F F
29372024-01-24T19:06:37.286Z 9 180-199 25 25 25 37 37 37 F F F
29382024-01-24T19:06:37.286Z 10 200-219 25 25 25 37 37 37 F F F
29392024-01-24T19:06:37.286Z 11 220-239 26 26 25 42 42 37 F F F <---
29402024-01-24T19:06:37.288Z 12 240-259 26 26 23 40 40 33 T T F <---
29412024-01-24T19:06:37.288Z 13 260-279 26 26 23 40 40 33 F F F <---
29422024-01-24T19:06:37.289Z 14 280-299 26 26 25 40 40 37 F F F <---
29432024-01-24T19:06:37.289Z 15 300-319 23 23 23 33 33 33 F F F
29442024-01-24T19:06:37.289Z 16 320-339 23 23 23 33 33 33 F F F
29452024-01-24T19:06:37.289Z 17 340-359 26 26 10 40 40 13 F F F <---
29462024-01-24T19:06:37.289Z 18 360-379 26 26 18 40 40 24 T T F <---
29472024-01-24T19:06:37.289Z 19 380-399 23 23 23 33 33 33 T T F <---
29482024-01-24T19:06:37.289Z 20 400-419 25 25 25 37 37 37 F F F
29492024-01-24T19:06:37.290Z 21 420-439 22 22 22 32 32 32 F F F
29502024-01-24T19:06:37.290Z 22 440-459 23 23 23 33 33 33 F F F
29512024-01-24T19:06:37.290Z 23 460-479 26 26 23 40 40 33 F F F <---
29522024-01-24T19:06:37.290Z 24 480-499 26 26 25 40 40 37 F F F <---
29532024-01-24T19:06:37.290Z 25 500-519 26 26 26 42 42 38 F F F <---
29542024-01-24T19:06:37.291Z 26 520-539 26 26 25 42 42 37 F F F <---
29552024-01-24T19:06:37.291Z 27 540-559 25 25 25 37 37 37 F F F
29562024-01-24T19:06:37.291Z 28 560-579 23 23 23 33 33 33 F F F
29572024-01-24T19:06:37.292Z 29 580-599 26 26 26 38 38 38 F F F
29582024-01-24T19:06:37.292ZMax gen: 26, Max flush: 42
29592024-01-24T19:06:37.292ZError: Difference in extent metadata found!
29602024-01-24T19:06:37.292ZA Difference in extent metadata is expected here
29612024-01-24T19:06:37.292ZOn loop 13
29622024-01-24T19:06:37.292Z
29632024-01-24T19:06:37.292ZVerifying data now
29642024-01-24T19:06:50.994ZNo change after verify
29652024-01-24T19:06:50.994ZLoop: 13 Downstairs dump after verify (and repair):
29662024-01-24T19:06:50.994ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29672024-01-24T19:06:50.994Z 0 000-019 23 23 23 33 33 33 F F F
29682024-01-24T19:06:50.995Z 1 020-039 24 24 24 34 34 34 F F F
29692024-01-24T19:06:50.995Z 2 040-059 26 26 26 40 40 40 F F F
29702024-01-24T19:06:50.995Z 3 060-079 26 26 26 40 40 40 F F F
29712024-01-24T19:06:50.995Z 4 080-099 19 19 19 25 25 25 F F F
29722024-01-24T19:06:50.995Z 5 100-119 27 27 27 43 43 43 F F F
29732024-01-24T19:06:50.995Z 6 120-139 21 21 21 30 30 30 F F F
29742024-01-24T19:06:50.995Z 7 140-159 22 22 22 32 32 32 F F F
29752024-01-24T19:06:50.995Z 8 160-179 23 23 23 33 33 33 F F F
29762024-01-24T19:06:50.995Z 9 180-199 25 25 25 37 37 37 F F F
29772024-01-24T19:06:51.333Z 10 200-219 25 25 25 37 37 37 F F F
29782024-01-24T19:06:51.333Z 11 220-239 26 26 26 42 42 42 F F F
29792024-01-24T19:06:51.334Z 12 240-259 27 27 27 43 43 43 F F F
29802024-01-24T19:06:51.334Z 13 260-279 26 26 26 40 40 40 F F F
29812024-01-24T19:06:51.334Z 14 280-299 26 26 26 40 40 40 F F F
29822024-01-24T19:06:51.335Z 15 300-319 23 23 23 33 33 33 F F F
29832024-01-24T19:06:51.336Z 16 320-339 23 23 23 33 33 33 F F F
29842024-01-24T19:06:51.336Z 17 340-359 26 26 26 40 40 40 F F F
29852024-01-24T19:06:51.336Z 18 360-379 27 27 27 43 43 43 F F F
29862024-01-24T19:06:51.340Z 19 380-399 27 27 27 43 43 43 F F F
29872024-01-24T19:06:51.341Z 20 400-419 25 25 25 37 37 37 F F F
29882024-01-24T19:06:51.341Z 21 420-439 22 22 22 32 32 32 F F F
29892024-01-24T19:06:51.341Z 22 440-459 23 23 23 33 33 33 F F F
29902024-01-24T19:06:51.341Z 23 460-479 26 26 26 40 40 40 F F F
29912024-01-24T19:06:51.341Z 24 480-499 26 26 26 40 40 40 F F F
29922024-01-24T19:06:51.342Z 25 500-519 26 26 26 42 42 42 F F F
29932024-01-24T19:06:51.342Z 26 520-539 26 26 26 42 42 42 F F F
29942024-01-24T19:06:51.342Z 27 540-559 25 25 25 37 37 37 F F F
29952024-01-24T19:06:51.342Z 28 560-579 23 23 23 33 33 33 F F F
29962024-01-24T19:06:51.343Z 29 580-599 26 26 26 38 38 38 F F F
29972024-01-24T19:06:51.343ZMax gen: 27, Max flush: 43
29982024-01-24T19:06:51.343Z
29992024-01-24T19:06:51.343ZBegin loop 14
30002024-01-24T19:06:51.343ZDownstairs to restart: 1
30012024-01-24T19:06:51.343Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.103920278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241}
30022024-01-24T19:06:51.344Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.104432153Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241}
30032024-01-24T19:06:51.344Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.104486747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241}
30042024-01-24T19:06:51.344Z{"msg":"Crucible 97405d6b-0821-465a-987b-9be5e3ee558a has session id: 6d61c683-8dbd-442a-b9a9-95dc90ef5e87","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.104532198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30052024-01-24T19:06:51.345Z{"msg":"Upstairs opts: Upstairs UUID: 97405d6b-0821-465a-987b-9be5e3ee558a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.104597158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30062024-01-24T19:06:51.345Z{"msg":"Crucible stats registered with UUID: 97405d6b-0821-465a-987b-9be5e3ee558a","v":0,"name":"crucible","level":30Crucible runtime is spawned
30072024-01-24T19:06:51.345Z,"time":"2024-01-24T19:06:40.104696368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30082024-01-24T19:06:51.345Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.104916684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241}
30092024-01-24T19:06:51.346Z{"msg":"97405d6b-0821-465a-987b-9be5e3ee558a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105077013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30102024-01-24T19:06:51.346Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105145612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30112024-01-24T19:06:51.346Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105220495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30122024-01-24T19:06:51.347Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105326913Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30132024-01-24T19:06:51.347Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105512223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30142024-01-24T19:06:51.347Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105635833Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30152024-01-24T19:06:51.348Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105728605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30162024-01-24T19:06:51.348Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.10585636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30172024-01-24T19:06:51.351Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.105952622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30182024-01-24T19:06:51.351Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106039641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30192024-01-24T19:06:51.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106119381Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30202024-01-24T19:06:51.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106208832Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30212024-01-24T19:06:51.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106302203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30222024-01-24T19:06:51.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106825451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30232024-01-24T19:06:51.352Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.106942137Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30242024-01-24T19:06:51.353Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.107021429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30252024-01-24T19:06:51.353Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.107078395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30262024-01-24T19:06:51.353Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.10715946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30272024-01-24T19:06:51.353Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.107245364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30282024-01-24T19:06:51.353Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.107973759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30292024-01-24T19:06:51.353Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.10821616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30302024-01-24T19:06:51.353Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.108312471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30312024-01-24T19:06:51.354Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.108710686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30322024-01-24T19:06:51.354Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.108834835Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30332024-01-24T19:06:51.354Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.108899556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30342024-01-24T19:06:51.354Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109236265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30352024-01-24T19:06:51.356Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109326548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30362024-01-24T19:06:51.357Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109410657Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30372024-01-24T19:06:51.357Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109483645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30382024-01-24T19:06:51.357Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109557556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30392024-01-24T19:06:51.358Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 40, 40, 25, 43, 30, 32, 33, 37, 37, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109635791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30402024-01-24T19:06:51.358Z{"msg":"[0]R generation[0..12]: [23, 24, 26, 26, 19, 27, 21, 22, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109710702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30412024-01-24T19:06:51.358Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109788482Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30422024-01-24T19:06:51.358Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 40, 40, 25, 43, 30, 32, 33, 37, 37, 42]",Wait for a query_work_queue command to finish before sending IO
30432024-01-24T19:06:51.359Z"v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.109949875Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30442024-01-24T19:06:51.359Z{"msg":"[1]R generation[0..12]: [23, 24, 26, 26, 19, 27, 21, 22, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110032202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30452024-01-24T19:06:51.359Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110112153Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30462024-01-24T19:06:51.359Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 40, 40, 25, 43, 30, 32, 33, 37, 37, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110190564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30472024-01-24T19:06:51.360Z{"msg":"[2]R generation[0..12]: [23, 24, 26, 26, 19, 27, 21, 22, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110266003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30482024-01-24T19:06:51.360Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110342051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30492024-01-24T19:06:51.360Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110421437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30502024-01-24T19:06:51.360Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110502139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30512024-01-24T19:06:51.360Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110580417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30522024-01-24T19:06:51.361Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110652611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30532024-01-24T19:06:51.361Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110712496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30542024-01-24T19:06:51.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30552024-01-24T19:06:51.376Z,"time":"2024-01-24T19:06:40.1107772Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30562024-01-24T19:06:51.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110884233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30572024-01-24T19:06:51.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110936605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30582024-01-24T19:06:51.376Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.110983691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30592024-01-24T19:06:51.376Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.111028397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30602024-01-24T19:06:51.376Z{"msg":"97405d6b-0821-465a-987b-9be5e3ee558a is now active with session: 6d61c683-8dbd-442a-b9a9-95dc90ef5e87","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.11107421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30612024-01-24T19:06:51.377Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.111118844Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241,"session_id":"6d61c683-8dbd-442a-b9a9-95dc90ef5e87"}
30622024-01-24T19:06:51.377Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:40.111163915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1241}
30632024-01-24T19:06:51.377ZLoading write count information from file "/tmp/test_repair_verify.data"
30642024-01-24T19:06:51.377ZRun Repair workload
30652024-01-24T19:06:51.377ZError: Difference in extent metadata found!
30662024-01-24T19:06:51.377Z01/30 Write block 337 len 8192 data: 5 5
30672024-01-24T19:06:51.377Z02/30 Read block 157 len 28672
30682024-01-24T19:06:51.377Z03/30 Read block 460 len 4096
30692024-01-24T19:06:51.377Z04/30 Read block 21 len 16384
30702024-01-24T19:06:51.379Z05/30 Write block 135 len 28672 data: 3 3 3 3 4 4 5
30712024-01-24T19:06:51.379Z06/30 Flush
30722024-01-24T19:06:51.379Z07/30 Write block 75 len 28672 data: 3 2 3 3 3 3 4
30732024-01-24T19:06:51.379Z08/30 Write block 393 len 36864 data: 5 5 5 5 4 3 3 4 4
30742024-01-24T19:06:51.379Z09/30 Write block 372 len 16384 data: 2 3 3 3
30752024-01-24T19:06:51.379Z10/30 Read block 330 len 40960
30762024-01-24T19:06:51.379Z11/30 Read block 564 len 32768
30772024-01-24T19:06:51.423Z12/30 Flush
30782024-01-24T19:06:51.423Z13/30 Write block 371 len 24576 data: 3 3 4 4 4 3
30792024-01-24T19:06:51.423Z14/30 Write block 507 len 28672 data: 3 3 3 3 3 3 4
30802024-01-24T19:06:51.423Z15/30 Write block 317 len 36864 data: 5 5 5 5 4 4 5 4 5
30812024-01-24T19:06:51.424Z16/30 Write block 443 len 40960 data: 2 2 2 2 3 4 4 3 3 3
30822024-01-24T19:06:51.424Z17/30 Write block 442 len 28672 data: 2 3 3 3 3 4 5
30832024-01-24T19:06:51.424Z18/30 Read block 172 len 32768
30842024-01-24T19:06:51.424Z19/30 Read block 298 len 16384
30852024-01-24T19:06:51.424Z20/30 Read block 85 len 4096
30862024-01-24T19:06:51.424Z21/30 Read block 410 len 8192
30872024-01-24T19:06:51.424Z22/30 Read block 420 len 40960
30882024-01-24T19:06:51.424Z23/30 Read block 505 len 20480
30892024-01-24T19:06:51.425Z24/30 Write block 74 len 4096 data: 3
30902024-01-24T19:06:51.425Z25/30 Write block 93 len 20480 data: 2 2 2 3 4
30912024-01-24T19:06:51.425Z26/30 Read block 320 len 24576
30922024-01-24T19:06:51.425Z27/30 Read block 543 len 36864
30932024-01-24T19:06:51.425Z28/30 Write block 449 len 24576 data: 5 4 4 4 4 4
30942024-01-24T19:06:51.425Z29/30 Read block 302 len 12288
30952024-01-24T19:06:51.426Z30/30 Read block 173 len 16384
30962024-01-24T19:06:51.426Z----------------------------------------------------------------
30972024-01-24T19:06:51.426Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
30982024-01-24T19:06:51.426ZStates: Active Active Active
30992024-01-24T19:06:51.426ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31002024-01-24T19:06:51.426Z 1 Acked 1000 Write 2 Done Done Done false
31012024-01-24T19:06:51.426Z 2 Acked 1001 Read 7 Done Sent Done false
31022024-01-24T19:06:51.426Z 3 Acked 1002 Read 1 Done Sent Done false
31032024-01-24T19:06:51.426Z 4 Acked 1003 Read 4 Done Sent Done false
31042024-01-24T19:06:51.426Z 5 Acked 1004 Write 7 Done Sent Done false
31052024-01-24T19:06:51.426Z 6 Acked 1005 Flush 0 Done Sent Done false
31062024-01-24T19:06:51.427Z 7 Acked 1006 Write 7 Done Sent Done false
31072024-01-24T19:06:51.427Z 8 Acked 1007 Write 9 Done Sent Done false
31082024-01-24T19:06:51.427Z 9 Acked 1008 Write 4 Done Sent Done false
31092024-01-24T19:06:51.427Z 10 Acked 1009 Read 10 Done Sent Done false
31102024-01-24T19:06:51.427Z 11 Acked 1010 Read 8 Done Sent Done false
31112024-01-24T19:06:51.427Z 12 Acked 1011 Flush 0 Done Sent Done false
31122024-01-24T19:06:51.427Z 13 Acked 1012 Write 6 Done Sent Done false
31132024-01-24T19:06:51.427Z 14 Acked 1013 Write 7 Done Sent Done false
31142024-01-24T19:06:51.427Z 15 Acked 1014 Write 9 Done Sent Done false
31152024-01-24T19:06:51.427Z 16 Acked 1015 Write 10 Done Sent Done false
31162024-01-24T19:06:51.427Z 17 Acked 1016 Write 7 Done Sent Done false
31172024-01-24T19:06:51.427Z 18 Acked 1017 Read 8 Done Sent Done false
31182024-01-24T19:06:51.429Z 19 Acked 1018 Read 4 Done Sent Done false
31192024-01-24T19:06:51.429Z 20 Acked 1019 Read 1 Done Sent Done false
31202024-01-24T19:06:51.429Z 21 Acked 1020 Read 2 Done Sent Done false
31212024-01-24T19:06:51.429Z 22 Acked 1021 Read 10 Done Sent Done false
31222024-01-24T19:06:51.429Z 23 Acked 1022 Read 5 Done Sent Done false
31232024-01-24T19:06:51.429Z 24 Acked 1023 Write 1 Done Sent Done false
31242024-01-24T19:06:51.429Z 25 Acked 1024 Write 5 Done Sent Done false
31252024-01-24T19:06:51.429Z 26 Acked 1025 Read 6 Done Sent Done false
31262024-01-24T19:06:51.429Z 27 Acked 1026 Read 9 Done Sent Done false
31272024-01-24T19:06:51.429Z 28 Acked 1027 Write 6 Done Sent Done false
31282024-01-24T19:06:51.429Z 29 Acked 1028 Read 3 Done Sent Done false
31292024-01-24T19:06:51.429Z 30 Acked 1029 Read 4 Sent Sent Done false
31302024-01-24T19:06:51.429Z STATES DS:0 DS:1 DS:2 TOTAL
31312024-01-24T19:06:51.429Z New 0 0 0 0
31322024-01-24T19:06:51.429Z Sent 1 29 0 30
31332024-01-24T19:06:51.429Z Done 29 1 30 60
31342024-01-24T19:06:51.429Z Skipped 0 0 0 0
31352024-01-24T19:06:51.430Z Error 0 0 0 0
31362024-01-24T19:06:51.430ZLast Flush: 1011 0 1011
31372024-01-24T19:06:51.430ZDownstairs last five completed:
31382024-01-24T19:06:51.430ZUpstairs last five completed: 30 29 28 27 26
31392024-01-24T19:06:51.430ZWrote out file "/tmp/test_repair_verify.data"
31402024-01-24T19:06:51.430Z
31412024-01-24T19:06:51.430ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31422024-01-24T19:06:51.430ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31432024-01-24T19:06:51.431Z 0 000-019 23 23 23 33 33 33 F F F
31442024-01-24T19:06:51.431Z 1 020-039 24 24 24 34 34 34 F F F
31452024-01-24T19:06:51.431Z 2 040-059 26 26 26 40 40 40 F F F
31462024-01-24T19:06:51.431Z 3 060-079 28 26 28 45 40 45 T F T <---
31472024-01-24T19:06:51.431Z 4 080-099 28 19 28 45 25 45 T F T <---
31482024-01-24T19:06:51.431Z 5 100-119 27 27 27 43 43 43 F F F
31492024-01-24T19:06:51.431Z 6 120-139 28 21 28 44 30 44 F F F <---
31502024-01-24T19:06:51.431Z 7 140-159 28 22 28 44 32 44 F F F <---
31512024-01-24T19:06:51.431Z 8 160-179 23 23 23 33 33 33 F F F
31522024-01-24T19:06:51.431Z 9 180-199 25 25 25 37 37 37 F F F
31532024-01-24T19:06:51.431Z 10 200-219 25 25 25 37 37 37 F F F
31542024-01-24T19:06:51.431Z 11 220-239 26 26 26 42 42 42 F F F
31552024-01-24T19:06:51.431Z 12 240-259 27 27 27 43 43 43 F F F
31562024-01-24T19:06:51.431Z 13 260-279 26 26 26 40 40 40 F F F
31572024-01-24T19:06:51.431Z 14 280-299 26 26 26 40 40 40 F F F
31582024-01-24T19:06:51.431Z 15 300-319 23 23 23 33 33 33 T F T <---
31592024-01-24T19:06:51.431Z 16 320-339 28 23 28 44 33 44 T T T <---
31602024-01-24T19:06:51.431Z 17 340-359 26 26 26 40 40 40 F F F
31612024-01-24T19:06:51.432Z 18 360-379 28 27 28 45 43 45 T F T <---
31622024-01-24T19:06:51.432Z 19 380-399 28 27 28 45 43 45 F F F <---
31632024-01-24T19:06:51.432Z 20 400-419 28 25 28 45 37 45 F F F <---
31642024-01-24T19:06:51.432Z 21 420-439 22 22 22 32 32 32 F F F
31652024-01-24T19:06:51.432Z 22 440-459 23 23 23 33 33 33 T F T <---
31662024-01-24T19:06:51.432Z 23 460-479 26 26 26 40 40 40 F F F
31672024-01-24T19:06:51.432Z 24 480-499 26 26 26 40 40 40 F F F
31682024-01-24T19:06:51.432Z 25 500-519 26 26 26 42 42 42 T F T <---
31692024-01-24T19:06:51.432Z 26 520-539 26 26 26 42 42 42 F F F
31702024-01-24T19:06:51.432Z 27 540-559 25 25 25 37 37 37 F F F
31712024-01-24T19:06:51.432Z 28 560-579 23 23 23 33 33 33 F F F
31722024-01-24T19:06:51.432Z 29 580-599 26 26 26 38 38 38 F F F
31732024-01-24T19:06:51.432ZMax gen: 28, Max flush: 45
31742024-01-24T19:06:51.432ZA Difference in extent metadata is expected here
31752024-01-24T19:06:51.432ZOn loop 14
31762024-01-24T19:06:51.432Z
31772024-01-24T19:06:51.457ZVerifying data now
31782024-01-24T19:06:51.457ZNo change after verify
31792024-01-24T19:06:51.457ZLoop: 14 Downstairs dump after verify (and repair):
31802024-01-24T19:06:51.457ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31812024-01-24T19:06:51.458Z 0 000-019 23 23 23 33 33 33 F F F
31822024-01-24T19:06:51.458Z 1 020-039 24 24 24 34 34 34 F F F
31832024-01-24T19:06:51.458Z 2 040-059 26 26 26 40 40 40 F F F
31842024-01-24T19:06:51.458Z 3 060-079 29 29 29 46 46 46 F F F
31852024-01-24T19:06:51.458Z 4 080-099 29 29 29 46 46 46 F F F
31862024-01-24T19:06:51.459Z 5 100-119 27 27 27 43 43 43 F F F
31872024-01-24T19:06:51.459Z 6 120-139 28 28 28 44 44 44 F F F
31882024-01-24T19:06:51.459Z 7 140-159 28 28 28 44 44 44 F F F
31892024-01-24T19:06:51.459Z 8 160-179 23 23 23 33 33 33 F F F
31902024-01-24T19:06:51.459Z 9 180-199 25 25 25 37 37 37 F F F
31912024-01-24T19:06:51.459Z 10 200-219 25 25 25 37 37 37 F F F
31922024-01-24T19:06:51.461Z 11 220-239 26 26 26 42 42 42 F F F
31932024-01-24T19:06:51.461Z 12 240-259 27 27 27 43 43 43 F F F
31942024-01-24T19:06:51.461Z 13 260-279 26 26 26 40 40 40 F F F
31952024-01-24T19:06:51.461Z 14 280-299 26 26 26 40 40 40 F F F
31962024-01-24T19:06:51.461Z 15 300-319 29 29 29 46 46 46 F F F
31972024-01-24T19:06:51.462Z 16 320-339 29 29 29 46 46 46 F F F
31982024-01-24T19:06:51.462Z 17 340-359 26 26 26 40 40 40 F F F
31992024-01-24T19:06:51.462Z 18 360-379 29 29 29 46 46 46 F F F
32002024-01-24T19:06:51.462Z 19 380-399 28 28 28 45 45 45 F F F
32012024-01-24T19:06:51.463Z 20 400-419 28 28 28 45 45 45 F F F
32022024-01-24T19:06:51.463Z 21 420-439 22 22 22 32 32 32 F F F
32032024-01-24T19:06:51.463Z 22 440-459 29 29 29 46 46 46 F F F
32042024-01-24T19:06:51.463Z 23 460-479 26 26 26 40 40 40 F F F
32052024-01-24T19:06:51.463Z 24 480-499 26 26 26 40 40 40 F F F
32062024-01-24T19:06:51.463Z 25 500-519 29 29 29 46 46 46 F F F
32072024-01-24T19:06:51.463Z 26 520-539 26 26 26 42 42 42 F F F
32082024-01-24T19:06:51.463Z 27 540-559 25 25 25 37 37 37 F F F
32092024-01-24T19:06:51.464Z 28 560-579 23 23 23 33 33 33 F F F
32102024-01-24T19:06:51.464Z 29 580-599 26 26 26 38 38 38 F F F
32112024-01-24T19:06:51.464ZMax gen: 29, Max flush: 46
32122024-01-24T19:06:51.464Z
32132024-01-24T19:06:51.465ZBegin loop 15
32142024-01-24T19:06:51.465ZDownstairs to restart: 2
32152024-01-24T19:06:51.465Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.266651035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249}
32162024-01-24T19:06:51.465Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267081921Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249}
32172024-01-24T19:06:51.487Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267111451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249}
32182024-01-24T19:06:51.487Z{"msg":"Crucible 9a13acaf-841b-4c76-905e-4d9057267d6a has session id: 4e8e8d03-f93b-48d9-86eb-7f57ba116ced","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267155099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32192024-01-24T19:06:51.497Z{"msg":"Upstairs opts: Upstairs UUID: 9a13acaf-841b-4c76-905e-4d9057267d6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267215791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32202024-01-24T19:06:51.497Z{"msg":"Crucible stats registered with UUID: 9a13acaf-841b-4c76-905e-4d9057267d6a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267256986Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32212024-01-24T19:06:51.497ZCrucible runtime is spawned
32222024-01-24T19:06:51.497Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.26745768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249}
32232024-01-24T19:06:51.497Z{"msg":"9a13acaf-841b-4c76-905e-4d9057267d6a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.26776808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32242024-01-24T19:06:51.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267800021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32252024-01-24T19:06:51.498Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267824588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32262024-01-24T19:06:51.498Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.26784643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32272024-01-24T19:06:51.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.26786861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32282024-01-24T19:06:51.499Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267891254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32292024-01-24T19:06:51.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267913881Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32302024-01-24T19:06:51.500Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267942261Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32312024-01-24T19:06:51.500Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267966113Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32322024-01-24T19:06:51.500Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.267988629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32332024-01-24T19:06:51.501Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.268345039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32342024-01-24T19:06:51.501Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.268397281Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32352024-01-24T19:06:51.501Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.268441885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32362024-01-24T19:06:51.501Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.269346623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32372024-01-24T19:06:51.503Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.269385567Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32382024-01-24T19:06:51.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.2696527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32392024-01-24T19:06:51.504Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.269693514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32402024-01-24T19:06:51.504Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.269879866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32412024-01-24T19:06:51.504Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.269930589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32422024-01-24T19:06:51.504Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.27060277Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32432024-01-24T19:06:51.505Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.27070024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32442024-01-24T19:06:51.505Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.271016891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32452024-01-24T19:06:51.505Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.271466751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32462024-01-24T19:06:51.505Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.271521844Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32472024-01-24T19:06:51.512Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:45.27156815Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32482024-01-24T19:06:51.512Z{Wait for a query_work_queue command to finish before sending IO
32492024-01-24T19:06:51.512Z"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.21546607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32502024-01-24T19:06:51.512Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.21552705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32512024-01-24T19:06:51.512Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215552104Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32522024-01-24T19:06:51.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215573222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32532024-01-24T19:06:51.513Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215599539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32542024-01-24T19:06:51.513Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 40, 46, 46, 43, 44, 44, 33, 37, 37, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215622334Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32552024-01-24T19:06:51.513Z{"msg":"[0]R generation[0..12]: [23, 24, 26, 29, 29, 27, 28, 28, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215644775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32562024-01-24T19:06:51.513Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215668556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32572024-01-24T19:06:51.515Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 40, 46, 46, 43, 44, 44, 33, 37, 37, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215692012Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32582024-01-24T19:06:51.515Z{"msg":"[1]R generation[0..12]: [23, 24, 26, 29, 29, 27, 28, 28, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215714599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32592024-01-24T19:06:51.515Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215737852Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32602024-01-24T19:06:51.516Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 40, 46, 46, 43, 44, 44, 33, 37, 37, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215771829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32612024-01-24T19:06:51.516Z{"msg":"[2]R generation[0..12]: [23, 24, 26, 29, 29, 27, 28, 28, 23, 25, 25, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215798298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32622024-01-24T19:06:51.516Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215848195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32632024-01-24T19:06:51.516Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215897044Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32642024-01-24T19:06:51.516Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215943848Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32652024-01-24T19:06:51.516Z{"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.215997261Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32662024-01-24T19:06:51.516Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216054615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32672024-01-24T19:06:51.517Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216100783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32682024-01-24T19:06:51.517Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216155648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32692024-01-24T19:06:51.517Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216205988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32702024-01-24T19:06:51.517Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.21625203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32712024-01-24T19:06:51.517ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32722024-01-24T19:06:51.517Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216302104Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32732024-01-24T19:06:51.517Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216346414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32742024-01-24T19:06:51.517Z{"msg":"9a13acaf-841b-4c76-905e-4d9057267d6a is now active with session: 4e8e8d03-f93b-48d9-86eb-7f57ba116ced","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216392426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32752024-01-24T19:06:51.517Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216436248Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249,"session_id":"4e8e8d03-f93b-48d9-86eb-7f57ba116ced"}
32762024-01-24T19:06:51.518Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:49.216480689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1249}
32772024-01-24T19:06:51.562ZLoading write count information from file "/tmp/test_repair_verify.data"
32782024-01-24T19:06:51.563ZRun Repair workload
32792024-01-24T19:06:51.563Z01/30 Write block 428 len 4096 data: 3
32802024-01-24T19:06:51.563Z02/30 Write block 180 len 40960 data: 4 4 4 3 4 5 5 4 4 3
32812024-01-24T19:06:51.563Z03/30 Read block 449 len 36864
32822024-01-24T19:06:51.564Z04/30 Write block 93 len 8192 data: 3 3
32832024-01-24T19:06:51.564Z05/30 Read block 281 len 16384
32842024-01-24T19:06:51.564Z06/30 Write block 453 len 24576 data: 5 5 4 4 4 6
32852024-01-24T19:06:51.564Z07/30 Read block 399 len 36864
32862024-01-24T19:06:51.564Z08/30 Write block 192 len 36864 data: 3 3 3 3 3 3 3 2 3
32872024-01-24T19:06:51.564Z09/30 Read block 380 len 20480
32882024-01-24T19:06:51.564Z10/30 Write block 349 len 12288 data: 2 2 2
32892024-01-24T19:06:51.564Z11/30 Read block 280 len 4096
32902024-01-24T19:06:51.564Z12/30 Read block 85 len 28672
32912024-01-24T19:06:51.564Z13/30 Write block 537 len 8192 data: 5 4
32922024-01-24T19:06:51.564Z14/30 Write block 519 len 40960 data: 5 5 5 4 3 4 5 5 4 4
32932024-01-24T19:06:51.565Z15/30 Read block 450 len 28672
32942024-01-24T19:06:51.565Z16/30 Flush
32952024-01-24T19:06:51.565Z17/30 Write block 74 len 40960 data: 4 4 3 4 4 4 4 5 3 4
32962024-01-24T19:06:51.565Z18/30 Write block 418 len 8192 data: 4 4
32972024-01-24T19:06:51.565Z19/30 Write block 496 len 12288 data: 4 3 4
32982024-01-24T19:06:51.565Z20/30 Read block 259 len 40960
32992024-01-24T19:06:51.565Z21/30 Read block 74 len 20480
33002024-01-24T19:06:51.565Z22/30 Read block 201 len 20480
33012024-01-24T19:06:51.566Z23/30 Read block 218 len 36864
33022024-01-24T19:06:51.566Z24/30 Write block 86 len 20480 data: 4 4 3 2 2
33032024-01-24T19:06:51.566Z25/30 Read block 348 len 28672
33042024-01-24T19:06:51.566Z26/30 Flush
33052024-01-24T19:06:51.566Z27/30 Write block 103 len 36864 data: 3 3 3 4 4 5 5 4 4
33062024-01-24T19:06:51.566Z28/30 Write block 359 len 24576 data: 4 4 4 4 4 4
33072024-01-24T19:06:51.566Z29/30 Write block 511 len 36864 data: 4 4 5 4 4 3 4 5 6
33082024-01-24T19:06:51.566Z30/30 Read block 512 len 28672
33092024-01-24T19:06:51.566Z----------------------------------------------------------------
33102024-01-24T19:06:51.567Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33112024-01-24T19:06:51.567ZStates: Active Active Active
33122024-01-24T19:06:51.567ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33132024-01-24T19:06:51.567Z 1 Acked 1000 Write 1 Done Done Done false
33142024-01-24T19:06:51.567Z 2 Acked 1001 Write 10 Done Done Done false
33152024-01-24T19:06:51.567Z 3 Acked 1002 Read 9 Done Done Done false
33162024-01-24T19:06:51.567Z 4 Acked 1003 Write 2 Done Done Sent false
33172024-01-24T19:06:51.568Z 5 Acked 1004 Read 4 Done Done Sent false
33182024-01-24T19:06:51.568Z 6 Acked 1005 Write 6 Done Done Sent false
33192024-01-24T19:06:51.569Z 7 Acked 1006 Read 9 Done Done Sent false
33202024-01-24T19:06:51.569Z 8 Acked 1007 Write 9 Done Done Sent false
33212024-01-24T19:06:51.569Z 9 Acked 1008 Read 5 Done Done Sent false
33222024-01-24T19:06:51.569Z 10 Acked 1009 Write 3 Done Done Sent false
33232024-01-24T19:06:51.569Z 11 Acked 1010 Read 1 Done Done Sent false
33242024-01-24T19:06:51.569Z 12 Acked 1011 Read 7 Done Done Sent false
33252024-01-24T19:06:51.569Z 13 Acked 1012 Write 2 Done Done Sent false
33262024-01-24T19:06:51.569Z 14 Acked 1013 Write 10 Done Done Sent false
33272024-01-24T19:06:51.569Z 15 Acked 1014 Read 7 Done Done Sent false
33282024-01-24T19:06:51.569Z 16 Acked 1015 Flush 0 Done Done Sent false
33292024-01-24T19:06:51.569Z 17 Acked 1016 Write 10 Done Done Sent false
33302024-01-24T19:06:51.569Z 18 Acked 1017 Write 2 Done Done Sent false
33312024-01-24T19:06:51.569Z 19 Acked 1018 Write 3 Done Done Sent false
33322024-01-24T19:06:51.569Z 20 Acked 1019 Read 10 Done Done Sent false
33332024-01-24T19:06:51.569Z 21 Acked 1020 Read 5 Done Done Sent false
33342024-01-24T19:06:51.569Z 22 Acked 1021 Read 5 Done Done Sent false
33352024-01-24T19:06:51.569Z 23 Acked 1022 Read 9 Done Done Sent false
33362024-01-24T19:06:51.569ZError: Difference in extent metadata found!
33372024-01-24T19:06:51.569Z 24 Acked 1023 Write 5 Done Done Sent false
33382024-01-24T19:06:51.569Z 25 Acked 1024 Read 7 Done Done Sent false
33392024-01-24T19:06:51.570Z 26 Acked 1025 Flush 0 Done Done Sent false
33402024-01-24T19:06:51.570Z 27 Acked 1026 Write 9 Done Done Sent false
33412024-01-24T19:06:51.570Z 28 Acked 1027 Write 6 Done Done Sent false
33422024-01-24T19:06:51.570Z 29 Acked 1028 Write 9 Done Done Sent false
33432024-01-24T19:06:51.570Z 30 Acked 1029 Read 7 Done Sent Sent false
33442024-01-24T19:06:51.570Z STATES DS:0 DS:1 DS:2 TOTAL
33452024-01-24T19:06:51.570Z New 0 0 0 0
33462024-01-24T19:06:51.570Z Sent 0 1 27 28
33472024-01-24T19:06:51.570Z Done 30 29 3 62
33482024-01-24T19:06:51.570Z Skipped 0 0 0 0
33492024-01-24T19:06:51.570Z Error 0 0 0 0
33502024-01-24T19:06:51.570ZLast Flush: 1025 1025 0
33512024-01-24T19:06:51.570ZDownstairs last five completed:
33522024-01-24T19:06:51.570ZUpstairs last five completed: 30 29 28 27 26
33532024-01-24T19:06:51.570ZWrote out file "/tmp/test_repair_verify.data"
33542024-01-24T19:06:51.570Z
33552024-01-24T19:06:51.570ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33562024-01-24T19:06:51.570ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33572024-01-24T19:06:51.570Z 0 000-019 23 23 23 33 33 33 F F F
33582024-01-24T19:06:51.571Z 1 020-039 24 24 24 34 34 34 F F F
33592024-01-24T19:06:51.571Z 2 040-059 26 26 26 40 40 40 F F F
33602024-01-24T19:06:51.571Z 3 060-079 30 30 29 48 48 46 F F F <---
33612024-01-24T19:06:51.571Z 4 080-099 30 30 29 48 48 46 F F F <---
33622024-01-24T19:06:51.571Z 5 100-119 27 27 27 43 43 43 T T F <---
33632024-01-24T19:06:51.571Z 6 120-139 28 28 28 44 44 44 F F F
33642024-01-24T19:06:51.571Z 7 140-159 28 28 28 44 44 44 F F F
33652024-01-24T19:06:51.571Z 8 160-179 23 23 23 33 33 33 F F F
33662024-01-24T19:06:51.571Z 9 180-199 30 30 25 47 47 37 F F T <---
33672024-01-24T19:06:51.572Z 10 200-219 30 30 25 47 47 37 F F F <---
33682024-01-24T19:06:51.572Z 11 220-239 26 26 26 42 42 42 F F F
33692024-01-24T19:06:51.572Z 12 240-259 27 27 27 43 43 43 F F F
33702024-01-24T19:06:51.572Z 13 260-279 26 26 26 40 40 40 F F F
33712024-01-24T19:06:51.572Z 14 280-299 26 26 26 40 40 40 F F F
33722024-01-24T19:06:51.572Z 15 300-319 29 29 29 46 46 46 F F F
33732024-01-24T19:06:51.572Z 16 320-339 29 29 29 46 46 46 F F F
33742024-01-24T19:06:51.572Z 17 340-359 30 30 26 47 47 40 T T F <---
33752024-01-24T19:06:51.572Z 18 360-379 29 29 29 46 46 46 T T F <---
33762024-01-24T19:06:51.572Z 19 380-399 28 28 28 45 45 45 F F F
33772024-01-24T19:06:51.588Z 20 400-419 30 30 28 48 48 45 F F F <---
33782024-01-24T19:06:51.588Z 21 420-439 30 30 22 47 47 32 F F T <---
33792024-01-24T19:06:51.588Z 22 440-459 30 30 29 47 47 46 F F F <---
33802024-01-24T19:06:51.588Z 23 460-479 26 26 26 40 40 40 F F F
33812024-01-24T19:06:51.588Z 24 480-499 30 30 26 48 48 40 F F F <---
33822024-01-24T19:06:51.588Z 25 500-519 30 30 29 47 47 46 T T F <---
33832024-01-24T19:06:51.588Z 26 520-539 30 30 26 47 47 42 F F F <---
33842024-01-24T19:06:51.588Z 27 540-559 25 25 25 37 37 37 F F F
33852024-01-24T19:06:51.588Z 28 560-579 23 23 23 33 33 33 F F F
33862024-01-24T19:06:51.588Z 29 580-599 26 26 26 38 38 38 F F F
33872024-01-24T19:06:51.589ZMax gen: 30, Max flush: 48
33882024-01-24T19:06:51.589ZA Difference in extent metadata is expected here
33892024-01-24T19:06:51.589ZOn loop 15
33902024-01-24T19:06:51.589Z
33912024-01-24T19:06:51.589ZVerifying data now
33922024-01-24T19:06:56.811ZNo change after verify
33932024-01-24T19:06:56.811ZLoop: 15 Downstairs dump after verify (and repair):
33942024-01-24T19:06:56.829ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33952024-01-24T19:06:56.829Z 0 000-019 23 23 23 33 33 33 F F F
33962024-01-24T19:06:56.849Z 1 020-039 24 24 24 34 34 34 F F F
33972024-01-24T19:06:56.849Z 2 040-059 26 26 26 40 40 40 F F F
33982024-01-24T19:06:56.849Z 3 060-079 30 30 30 48 48 48 F F F
33992024-01-24T19:06:56.850Z 4 080-099 30 30 30 48 48 48 F F F
34002024-01-24T19:06:56.850Z 5 100-119 31 31 31 49 49 49 F F F
34012024-01-24T19:06:56.850Z 6 120-139 28 28 28 44 44 44 F F F
34022024-01-24T19:06:56.850Z 7 140-159 28 28 28 44 44 44 F F F
34032024-01-24T19:06:56.850Z 8 160-179 23 23 23 33 33 33 F F F
34042024-01-24T19:06:56.851Z 9 180-199 30 30 30 47 47 47 F F F
34052024-01-24T19:06:56.851Z 10 200-219 30 30 30 47 47 47 F F F
34062024-01-24T19:06:56.851Z 11 220-239 26 26 26 42 42 42 F F F
34072024-01-24T19:06:56.851Z 12 240-259 27 27 27 43 43 43 F F F
34082024-01-24T19:06:56.851Z 13 260-279 26 26 26 40 40 40 F F F
34092024-01-24T19:06:56.851Z 14 280-299 26 26 26 40 40 40 F F F
34102024-01-24T19:06:56.852Z 15 300-319 29 29 29 46 46 46 F F F
34112024-01-24T19:06:56.852Z 16 320-339 29 29 29 46 46 46 F F F
34122024-01-24T19:06:56.852Z 17 340-359 31 31 31 49 49 49 F F F
34132024-01-24T19:06:56.852Z 18 360-379 31 31 31 49 49 49 F F F
34142024-01-24T19:06:56.852Z 19 380-399 28 28 28 45 45 45 F F F
34152024-01-24T19:06:56.853Z 20 400-419 30 30 30 48 48 48 F F F
34162024-01-24T19:06:56.853Z 21 420-439 30 30 30 47 47 47 F F F
34172024-01-24T19:06:56.853Z 22 440-459 30 30 30 47 47 47 F F F
34182024-01-24T19:06:56.853Z 23 460-479 26 26 26 40 40 40 F F F
34192024-01-24T19:06:56.854Z 24 480-499 30 30 30 48 48 48 F F F
34202024-01-24T19:06:56.854Z 25 500-519 31 31 31 49 49 49 F F F
34212024-01-24T19:06:56.855Z 26 520-539 30 30 30 47 47 47 F F F
34222024-01-24T19:06:56.855Z 27 540-559 25 25 25 37 37 37 F F F
34232024-01-24T19:06:56.855Z 28 560-579 23 23 23 33 33 33 F F F
34242024-01-24T19:06:56.856Z 29 580-599 26 26 26 38 38 38 F F F
34252024-01-24T19:06:56.856ZMax gen: 31, Max flush: 49
34262024-01-24T19:06:56.856Z
34272024-01-24T19:06:56.856ZBegin loop 16
34282024-01-24T19:06:56.856ZDownstairs to restart: 2
34292024-01-24T19:06:56.870Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520197175Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257}
34302024-01-24T19:06:56.871Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520668669Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257}
34312024-01-24T19:06:56.871Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520699282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257}
34322024-01-24T19:06:56.871Z{"msg":"Crucible 6e2be3d9-ce24-48ec-9520-6dea2f0e7f08 has session id: 6f52ca55-e8b7-43f1-99b9-2a20525043ed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520726571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34332024-01-24T19:06:56.871Z{"msg":"Upstairs opts: Upstairs UUID: 6e2be3d9-ce24-48ec-9520-6dea2f0e7f08, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520817244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34342024-01-24T19:06:56.873Z{"msg":"Crucible stats registered with UUID: 6e2be3d9-ce24-48ec-9520-6dea2f0e7f08","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.520845148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34352024-01-24T19:06:56.873ZCrucible runtime is spawned
34362024-01-24T19:06:56.873Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521179332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257}
34372024-01-24T19:06:56.873Z{"msg":"6e2be3d9-ce24-48ec-9520-6dea2f0e7f08 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521430539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34382024-01-24T19:06:56.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521489347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34392024-01-24T19:06:56.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521517697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34402024-01-24T19:06:56.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521544021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34412024-01-24T19:06:56.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521605929Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34422024-01-24T19:06:56.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521633956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34432024-01-24T19:06:56.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521691885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34442024-01-24T19:06:56.885Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521769029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34452024-01-24T19:06:56.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521909237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34462024-01-24T19:06:56.886Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.521990816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34472024-01-24T19:06:56.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.522036696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34482024-01-24T19:06:56.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.522189734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34492024-01-24T19:06:56.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.522276135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34502024-01-24T19:06:56.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.523373142Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34512024-01-24T19:06:56.886Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.523441096Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34522024-01-24T19:06:57.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.523690775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34532024-01-24T19:06:57.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52378696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34542024-01-24T19:06:57.345Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52434601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34552024-01-24T19:06:57.345Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52461776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34562024-01-24T19:06:57.345Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.524706139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34572024-01-24T19:06:57.345Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.524810006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34582024-01-24T19:06:57.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.525125315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34592024-01-24T19:06:57.345Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.525186116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34602024-01-24T19:06:57.346Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.525212838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34612024-01-24T19:06:57.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52567034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34622024-01-24T19:06:57.346Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.525771326Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34632024-01-24T19:06:57.350Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52584058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34642024-01-24T19:06:57.350Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.525868401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34652024-01-24T19:06:57.350Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526478057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34662024-01-24T19:06:57.351Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526536994Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34672024-01-24T19:06:57.351Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 49, 44, 44, 33, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526565332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34682024-01-24T19:06:57.351Z{"msg":"[0]R generation[0..12]: [23, 24, 26, 30, 30, 31, 28, 28, 23, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526627373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34692024-01-24T19:06:57.351Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526692064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34702024-01-24T19:06:57.351Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 49, 44, 44, 33, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526755477Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34712024-01-24T19:06:57.351Z{"msg":"[1]R generation[0..12]: [23, 24, 26, 30, 30, 31, 28, 28, 23, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526787697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34722024-01-24T19:06:57.352Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
34732024-01-24T19:06:57.352Zlevel":30,"time":"2024-01-24T19:06:55.526872323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34742024-01-24T19:06:57.352Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 49, 44, 44, 33, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.526936583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34752024-01-24T19:06:57.353Z{"msg":"[2]R generation[0..12]: [23, 24, 26, 30, 30, 31, 28, 28, 23, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52701816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34762024-01-24T19:06:57.353Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527076616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34772024-01-24T19:06:57.353Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527102787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34782024-01-24T19:06:57.353Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527173358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34792024-01-24T19:06:57.353Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527204361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34802024-01-24T19:06:57.353Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527265042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34812024-01-24T19:06:57.353Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527293208Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34822024-01-24T19:06:57.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527349966Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34832024-01-24T19:06:57.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527378007Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34842024-01-24T19:06:57.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.52744895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34852024-01-24T19:06:57.354Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527529537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34862024-01-24T19:06:57.354Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527571723Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34872024-01-24T19:06:57.355Z{"msg":"6e2be3d9-ce24-48ec-9520-6dea2f0e7f08 is now active with session: 6f52ca55-e8b7-43f1-99b9-2a20525043ed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527634297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34882024-01-24T19:06:57.355Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527675312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257,"session_id":"6f52ca55-e8b7-43f1-99b9-2a20525043ed"}
34892024-01-24T19:06:57.355Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:06:55.527731494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1257}
34902024-01-24T19:06:57.355ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
34912024-01-24T19:06:57.355ZLoading write count information from file "/tmp/test_repair_verify.data"
34922024-01-24T19:06:57.355ZRun Repair workload
34932024-01-24T19:06:57.355Z01/30 Write block 575 len 8192 data: 2 2
34942024-01-24T19:06:57.355Z02/30 Write block 557 len 28672 data: 3 3 3 3 3 3 3
34952024-01-24T19:06:57.363Z03/30 Write block 579 len 24576 data: 3 3 3 3 3 3
34962024-01-24T19:06:57.363Z04/30 Write block 585 len 12288 data: 3 3 3
34972024-01-24T19:06:57.363Z05/30 Write block 119 len 28672 data: 4 4 5 5 5 5 5
34982024-01-24T19:06:57.363Z06/30 Write block 579 len 24576 data: 4 4 4 4 4 4
34992024-01-24T19:06:57.363Z07/30 Read block 108 len 24576
35002024-01-24T19:06:57.363Z08/30 Read block 401 len 16384
35012024-01-24T19:06:57.364Z09/30 Flush
35022024-01-24T19:06:57.364Z10/30 Write block 103 len 40960 data: 4 4 4 5 5 6 6 5 5 4
35032024-01-24T19:06:57.364Z11/30 Write block 517 len 24576 data: 5 6 7 6 6 5
35042024-01-24T19:06:57.364Z12/30 Flush
35052024-01-24T19:06:57.364Z13/30 Write block 287 len 12288 data: 3 4 4
35062024-01-24T19:06:57.364Z14/30 Read block 172 len 36864
35072024-01-24T19:06:57.392Z15/30 Read block 588 len 40960
35082024-01-24T19:06:57.392Z16/30 Read block 131 len 16384
35092024-01-24T19:06:57.392Z17/30 Read block 1 len 4096
35102024-01-24T19:06:57.392Z18/30 Read block 16 len 20480
35112024-01-24T19:06:57.392Z19/30 Read block 510 len 24576
35122024-01-24T19:06:57.393Z20/30 Read block 148 len 36864
35132024-01-24T19:06:57.393Z21/30 Read block 542 len 24576
35142024-01-24T19:06:57.393Z22/30 Read block 356 len 4096
35152024-01-24T19:06:57.394Z23/30 Write block 321 len 20480 data: 5 5 6 5 6
35162024-01-24T19:06:57.394Z24/30 Write block 456 len 8192 data: 5 5
35172024-01-24T19:06:57.394Z25/30 Read block 110 len 36864
35182024-01-24T19:06:57.394Z26/30 Write block 162 len 16384 data: 4 4 3 3
35192024-01-24T19:06:57.394Z27/30 Read block 70 len 12288
35202024-01-24T19:06:57.394Z28/30 Write block 361 len 40960 data: 5 5 5 5 3 3 3 3 3 3
35212024-01-24T19:06:57.394Z29/30 Write block 365 len 32768 data: 4 4 4 4 4 4 4 4
35222024-01-24T19:06:57.414Z30/30 Read block 553 len 20480
35232024-01-24T19:06:57.415Z----------------------------------------------------------------
35242024-01-24T19:06:57.415Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35252024-01-24T19:06:57.416ZStates: Active Active Active
35262024-01-24T19:06:57.416ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35272024-01-24T19:06:57.416Z 1 Acked 1000 Write 2 Done Done Done false
35282024-01-24T19:06:57.465Z 2 Acked 1001 Write 7 Done Done Done false
35292024-01-24T19:06:57.465Z 3 Acked 1002 Write 6 Done Done Done false
35302024-01-24T19:06:57.465Z 4 Acked 1003 Write 3 Done Done Sent false
35312024-01-24T19:06:57.465Z 5 Acked 1004 Write 7 Done Done Sent false
35322024-01-24T19:06:57.465Z 6 Acked 1005 Write 6 Done Done Sent false
35332024-01-24T19:06:57.465Z 7 Acked 1006 Read 6 Done Done Sent false
35342024-01-24T19:06:57.466Z 8 Acked 1007 Read 4 Done Done Sent false
35352024-01-24T19:06:57.466Z 9 Acked 1008 Flush 0 Done Done Sent false
35362024-01-24T19:06:57.466Z 10 Acked 1009 Write 10 Done Done Sent false
35372024-01-24T19:06:57.466Z 11 Acked 1010 Write 6 Done Done Sent false
35382024-01-24T19:06:57.466Z 12 Acked 1011 Flush 0 Done Done Sent false
35392024-01-24T19:06:57.466Z 13 Acked 1012 Write 3 Done Done Sent false
35402024-01-24T19:06:57.496Z 14 Acked 1013 Read 9 Done Done Sent false
35412024-01-24T19:06:57.496Z 15 Acked 1014 Read 10 Done Done Sent false
35422024-01-24T19:06:57.496Z 16 Acked 1015 Read 4 Done Done Sent false
35432024-01-24T19:06:57.496Z 17 Acked 1016 Read 1 Done Done Sent false
35442024-01-24T19:06:57.499Z 18 Acked 1017 Read 5 Done Done Sent false
35452024-01-24T19:06:57.499Z 19 Acked 1018 Read 6 Done Done Sent false
35462024-01-24T19:06:57.499Z 20 Acked 1019 Read 9 Done Done Sent false
35472024-01-24T19:06:57.499Z 21 Acked 1020 Read 6 Done Done Sent false
35482024-01-24T19:06:57.499Z 22 Acked 1021 Read 1 Done Done Sent false
35492024-01-24T19:06:57.499Z 23 Acked 1022 Write 5 Done Done Sent false
35502024-01-24T19:06:57.499Z 24 Acked 1023 Write 2 Done Done Sent false
35512024-01-24T19:06:57.500Z 25 Acked 1024 Read 9 Done Done Sent false
35522024-01-24T19:06:57.500Z 26 Acked 1025 Write 4 Done Done Sent false
35532024-01-24T19:06:57.500Z 27 Acked 1026 Read 3 Done Done Sent false
35542024-01-24T19:06:57.500Z 28 Acked 1027 Write 10 Done Done Sent false
35552024-01-24T19:06:57.500Z 29 Acked 1028 Write 8 Done Done Sent false
35562024-01-24T19:06:57.500Z 30 Acked 1029 Read 5 Done Sent Sent false
35572024-01-24T19:06:57.500Z STATES DS:0 DS:1 DS:2 TOTAL
35582024-01-24T19:06:57.500Z New 0 0 0 0
35592024-01-24T19:06:57.500Z Sent 0 1 27 28
35602024-01-24T19:06:57.500Z Done 30 29 3 62
35612024-01-24T19:06:57.500Z Skipped 0 0 0 0
35622024-01-24T19:06:57.501Z Error 0 0 0 0
35632024-01-24T19:06:57.501ZLast Flush: 1011 1011 0
35642024-01-24T19:06:57.501ZDownstairs last five completed:
35652024-01-24T19:06:57.501ZUpstairs last five completed: 30 29 28 27 26
35662024-01-24T19:06:57.501ZWrote out file "/tmp/test_repair_verify.data"
35672024-01-24T19:06:57.501Z
35682024-01-24T19:06:57.501ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35692024-01-24T19:06:57.503ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35702024-01-24T19:06:57.503Z 0 000-019 23 23 23 33 33 33 F F F
35712024-01-24T19:06:57.503Z 1 020-039 24 24 24 34 34 34 F F F
35722024-01-24T19:06:57.504Z 2 040-059 26 26 26 40 40 40 F F F
35732024-01-24T19:06:57.504Z 3 060-079 30 30 30 48 48 48 F F F
35742024-01-24T19:06:57.504Z 4 080-099 30 30 30 48 48 48 F F F
35752024-01-24T19:06:57.504Z 5 100-119 32 32 31 51 51 49 F F F <---
35762024-01-24T19:06:57.504Z 6 120-139 32 32 28 50 50 44 F F F <---
35772024-01-24T19:06:57.505Z 7 140-159 28 28 28 44 44 44 F F F
35782024-01-24T19:06:57.505Z 8 160-179 23 23 23 33 33 33 T T F <---
35792024-01-24T19:06:57.505Z 9 180-199 30 30 30 47 47 47 F F F
35802024-01-24T19:06:57.505Z 10 200-219 30 30 30 47 47 47 F F F
35812024-01-24T19:06:57.505Z 11 220-239 26 26 26 42 42 42 F F F
35822024-01-24T19:06:57.505Z 12 240-259 27 27 27 43 43 43 F F F
35832024-01-24T19:06:57.505Z 13 260-279 26 26 26 40 40 40 F F F
35842024-01-24T19:06:57.506Z 14 280-299 26 26 26 40 40 40 T T F <---
35852024-01-24T19:06:57.506Z 15 300-319 29 29 29 46 46 46 F F F
35862024-01-24T19:06:57.506Z 16 320-339 29 29 29 46 46 46 T T F <---
35872024-01-24T19:06:57.506Z 17 340-359 31 31 31 49 49 49 F F F
35882024-01-24T19:06:57.507Z 18 360-379 31 31 31 49 49 49 T T F <---
35892024-01-24T19:06:57.507Z 19 380-399 28 28 28 45 45 45 F F F
35902024-01-24T19:06:57.507Z 20 400-419 30 30 30 48 48 48 F F F
35912024-01-24T19:06:57.507Z 21 420-439 30 30 30 47 47 47 F F F
35922024-01-24T19:06:57.507Z 22 440-459 30 30 30 47 47 47 T T F <---
35932024-01-24T19:06:57.508Z 23 460-479 26 26 26 40 40 40 F F F
35942024-01-24T19:06:57.508Z 24 480-499 30 30 30 48 48 48 F F F
35952024-01-24T19:06:57.508Z 25 500-519 32 32 31 51 51 49 F F F <---
35962024-01-24T19:06:57.508Z 26 520-539 32 32 30 51 51 47 F F F <---
35972024-01-24T19:06:57.508Z 27 540-559 32 32 25 50 50 37 F F T <---
35982024-01-24T19:06:57.508Z 28 560-579 32 32 23 50 50 33 F F T <---
35992024-01-24T19:06:57.509Z 29 580-599 32 32 26 50 50 38 F F T <---
36002024-01-24T19:06:57.509ZMax gen: 32, Max flush: 51
36012024-01-24T19:06:57.509ZError: Difference in extent metadata found!
36022024-01-24T19:06:57.509ZA Difference in extent metadata is expected here
36032024-01-24T19:06:57.509ZOn loop 16
36042024-01-24T19:06:57.509Z
36052024-01-24T19:06:57.509ZVerifying data now
36062024-01-24T19:07:03.044ZNo change after verify
36072024-01-24T19:07:03.044ZLoop: 16 Downstairs dump after verify (and repair):
36082024-01-24T19:07:03.066ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36092024-01-24T19:07:03.066Z 0 000-019 23 23 23 33 33 33 F F F
36102024-01-24T19:07:03.089Z 1 020-039 24 24 24 34 34 34 F F F
36112024-01-24T19:07:03.090Z 2 040-059 26 26 26 40 40 40 F F F
36122024-01-24T19:07:03.090Z 3 060-079 30 30 30 48 48 48 F F F
36132024-01-24T19:07:03.090Z 4 080-099 30 30 30 48 48 48 F F F
36142024-01-24T19:07:03.090Z 5 100-119 32 32 32 51 51 51 F F F
36152024-01-24T19:07:03.090Z 6 120-139 32 32 32 50 50 50 F F F
36162024-01-24T19:07:03.090Z 7 140-159 28 28 28 44 44 44 F F F
36172024-01-24T19:07:03.090Z 8 160-179 33 33 33 52 52 52 F F F
36182024-01-24T19:07:03.090Z 9 180-199 30 30 30 47 47 47 F F F
36192024-01-24T19:07:03.090Z 10 200-219 30 30 30 47 47 47 F F F
36202024-01-24T19:07:03.090Z 11 220-239 26 26 26 42 42 42 F F F
36212024-01-24T19:07:03.090Z 12 240-259 27 27 27 43 43 43 F F F
36222024-01-24T19:07:03.091Z 13 260-279 26 26 26 40 40 40 F F F
36232024-01-24T19:07:03.093Z 14 280-299 33 33 33 52 52 52 F F F
36242024-01-24T19:07:03.094Z 15 300-319 29 29 29 46 46 46 F F F
36252024-01-24T19:07:03.095Z 16 320-339 33 33 33 52 52 52 F F F
36262024-01-24T19:07:03.095Z 17 340-359 31 31 31 49 49 49 F F F
36272024-01-24T19:07:03.095Z 18 360-379 33 33 33 52 52 52 F F F
36282024-01-24T19:07:03.095Z 19 380-399 28 28 28 45 45 45 F F F
36292024-01-24T19:07:03.095Z 20 400-419 30 30 30 48 48 48 F F F
36302024-01-24T19:07:03.095Z 21 420-439 30 30 30 47 47 47 F F F
36312024-01-24T19:07:03.095Z 22 440-459 33 33 33 52 52 52 F F F
36322024-01-24T19:07:03.095Z 23 460-479 26 26 26 40 40 40 F F F
36332024-01-24T19:07:03.096Z 24 480-499 30 30 30 48 48 48 F F F
36342024-01-24T19:07:03.096Z 25 500-519 32 32 32 51 51 51 F F F
36352024-01-24T19:07:03.096Z 26 520-539 32 32 32 51 51 51 F F F
36362024-01-24T19:07:03.096Z 27 540-559 32 32 32 50 50 50 F F F
36372024-01-24T19:07:03.096Z 28 560-579 32 32 32 50 50 50 F F F
36382024-01-24T19:07:03.096Z 29 580-599 32 32 32 50 50 50 F F F
36392024-01-24T19:07:03.096ZMax gen: 33, Max flush: 52
36402024-01-24T19:07:03.096Z
36412024-01-24T19:07:03.097ZBegin loop 17
36422024-01-24T19:07:03.097ZDownstairs to restart: 2
36432024-01-24T19:07:03.181Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.754758637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265}
36442024-01-24T19:07:03.181Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755264451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265}
36452024-01-24T19:07:03.181Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755293421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265}
36462024-01-24T19:07:03.182Z{"msg":"Crucible 1ed2b8db-b19e-4733-a8cc-adedfabf4e47 has session id: 0ffffb9b-8b82-40e5-b32f-48041e4d804b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755338617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36472024-01-24T19:07:03.182Z{"msg":"Upstairs opts: Upstairs UUID: 1ed2b8db-b19e-4733-a8cc-adedfabf4e47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755384774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36482024-01-24T19:07:03.182ZCrucible runtime is spawned
36492024-01-24T19:07:03.182Z{"msg":"Crucible stats registered with UUID: 1ed2b8db-b19e-4733-a8cc-adedfabf4e47","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75555294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36502024-01-24T19:07:03.182Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755650256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265}
36512024-01-24T19:07:03.182Z{"msg":"1ed2b8db-b19e-4733-a8cc-adedfabf4e47 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.755940564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36522024-01-24T19:07:03.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756083393Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36532024-01-24T19:07:03.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756236571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36542024-01-24T19:07:03.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756310148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36552024-01-24T19:07:03.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756388053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36562024-01-24T19:07:03.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756462481Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36572024-01-24T19:07:03.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756508828Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36582024-01-24T19:07:03.189Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75655308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36592024-01-24T19:07:03.189Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756635711Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36602024-01-24T19:07:03.189Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756683945Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36612024-01-24T19:07:03.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756727127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36622024-01-24T19:07:03.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75680886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36632024-01-24T19:07:03.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.756853553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36642024-01-24T19:07:03.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.758313763Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36652024-01-24T19:07:03.190Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75838822Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36662024-01-24T19:07:03.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.758417291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36672024-01-24T19:07:03.190Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.758442594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36682024-01-24T19:07:03.190Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.758486824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36692024-01-24T19:07:03.190Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75851284Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36702024-01-24T19:07:03.190Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.759523993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36712024-01-24T19:07:03.191Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.759751286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36722024-01-24T19:07:03.191Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.75987599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36732024-01-24T19:07:03.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.760444677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36742024-01-24T19:07:03.191Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.760505185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36752024-01-24T19:07:03.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.760532626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36762024-01-24T19:07:03.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.760949184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36772024-01-24T19:07:03.192Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761009931Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36782024-01-24T19:07:03.192Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.76103686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36792024-01-24T19:07:03.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761059598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36802024-01-24T19:07:03.192Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761103933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36812024-01-24T19:07:03.192Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 51, 50, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761130556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36822024-01-24T19:07:03.193Z{"msg":"[0]R generation[0..12]: [23, 24, 26, 30, 30, 32, 32, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.76117421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36832024-01-24T19:07:03.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761202077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36842024-01-24T19:07:03.193Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 51, 50, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761225365Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36852024-01-24T19:07:03.195Z{"msg":"[1]R generation[0..12]: [23, 24, 26, 30, 30, 32, 32, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761268828Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36862024-01-24T19:07:03.195Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761295958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36872024-01-24T19:07:03.196Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 40, 48, 48, 51, 50, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761319545Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36882024-01-24T19:07:03.196Z{"msg":"[2]R generation[0..12]: [23, 24, 26, 30, 30, 32, 32, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761342692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36892024-01-24T19:07:03.196Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
36902024-01-24T19:07:03.196Z,"time":"2024-01-24T19:07:01.761440687Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36912024-01-24T19:07:03.196Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761570698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36922024-01-24T19:07:03.196Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761600616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36932024-01-24T19:07:03.196Z{"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761622319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36942024-01-24T19:07:03.197Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761645739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36952024-01-24T19:07:03.197Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761667936Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36962024-01-24T19:07:03.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761687741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36972024-01-24T19:07:03.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761737975Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36982024-01-24T19:07:03.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761763298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
36992024-01-24T19:07:03.197Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.76178534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
37002024-01-24T19:07:03.198Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761827035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
37012024-01-24T19:07:03.198Z{"msg":"1ed2b8db-b19e-4733-a8cc-adedfabf4e47 is now active with session: 0ffffb9b-8b82-40e5-b32f-48041e4d804b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761851659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
37022024-01-24T19:07:03.198Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761871941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265,"session_id":"0ffffb9b-8b82-40e5-b32f-48041e4d804b"}
37032024-01-24T19:07:03.198Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:01.761892132Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1265}
37042024-01-24T19:07:03.198ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37052024-01-24T19:07:03.198ZLoading write count information from file "/tmp/test_repair_verify.data"
37062024-01-24T19:07:03.198ZRun Repair workload
37072024-01-24T19:07:03.235Z01/30 Write block 123 len 40960 data: 6 6 6 4 3 2 3 3 2 2
37082024-01-24T19:07:03.236Z02/30 Write block 107 len 8192 data: 6 7
37092024-01-24T19:07:03.236Z03/30 Flush
37102024-01-24T19:07:03.236Z04/30 Write block 401 len 12288 data: 5 4 3
37112024-01-24T19:07:03.236Z05/30 Read block 120 len 12288
37122024-01-24T19:07:03.236Z06/30 Read block 449 len 28672
37132024-01-24T19:07:03.236Z07/30 Read block 572 len 40960
37142024-01-24T19:07:03.236Z08/30 Flush
37152024-01-24T19:07:03.236Z09/30 Write block 339 len 12288 data: 5 5 5
37162024-01-24T19:07:03.236Z10/30 Read block 194 len 16384
37172024-01-24T19:07:03.236Z11/30 Read block 441 len 24576
37182024-01-24T19:07:03.236Z12/30 Read block 130 len 40960
37192024-01-24T19:07:03.236Z13/30 Read block 209 len 8192
37202024-01-24T19:07:03.236Z14/30 Write block 591 len 8192 data: 4 6
37212024-01-24T19:07:03.237Z15/30 Read block 147 len 16384
37222024-01-24T19:07:03.237Z16/30 Read block 397 len 8192
37232024-01-24T19:07:03.237Z17/30 Read block 18 len 4096
37242024-01-24T19:07:03.237Z18/30 Write block 124 len 4096 data: 7
37252024-01-24T19:07:03.238Z19/30 Read block 361 len 4096
37262024-01-24T19:07:03.238Z20/30 Read block 66 len 12288
37272024-01-24T19:07:03.238Z21/30 Write block 1 len 28672 data: 2 2 3 4 4 4 4
37282024-01-24T19:07:03.238Z22/30 Read block 100 len 20480
37292024-01-24T19:07:03.238Z23/30 Write block 474 len 32768 data: 2 3 3 3 3 3 4 5
37302024-01-24T19:07:03.238Z24/30 Flush
37312024-01-24T19:07:03.238Z25/30 Write block 415 len 40960 data: 3 3 4 5 5 3 3 4 3 3
37322024-01-24T19:07:03.238Z26/30 Read block 576 len 24576
37332024-01-24T19:07:03.238Z27/30 Write block 325 len 8192 data: 7 5
37342024-01-24T19:07:03.238Z28/30 Read block 474 len 40960
37352024-01-24T19:07:03.238Z29/30 Read block 425 len 8192
37362024-01-24T19:07:03.238Z30/30 Write block 329 len 28672 data: 3 2 2 3 3 3 3
37372024-01-24T19:07:03.238Z----------------------------------------------------------------
37382024-01-24T19:07:03.238Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:27
37392024-01-24T19:07:03.238ZStates: Active Active Active
37402024-01-24T19:07:03.238ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37412024-01-24T19:07:03.238Z 4 Acked 1003 Write 3 Done Done Sent false
37422024-01-24T19:07:03.239Z 5 Acked 1004 Read 3 Done Done Sent false
37432024-01-24T19:07:03.239Z 6 Acked 1005 Read 7 Done Done Sent false
37442024-01-24T19:07:03.239Z 7 Acked 1006 Read 10 Done Done Sent false
37452024-01-24T19:07:03.239Z 8 Acked 1007 Flush 0 Done Done Sent false
37462024-01-24T19:07:03.239Z 9 Acked 1008 Write 3 Done Done Sent false
37472024-01-24T19:07:03.239Z 10 Acked 1009 Read 4 Done Done Sent false
37482024-01-24T19:07:03.239Z 11 Acked 1010 Read 6 Done Done Sent false
37492024-01-24T19:07:03.239Z 12 Acked 1011 Read 10 Done Done Sent false
37502024-01-24T19:07:03.239Z 13 Acked 1012 Read 2 Done Done Sent false
37512024-01-24T19:07:03.239Z 14 Acked 1013 Write 2 Done Done Sent false
37522024-01-24T19:07:03.239Z 15 Acked 1014 Read 4 Done Done Sent false
37532024-01-24T19:07:03.239Z 16 Acked 1015 Read 2 Done Done Sent false
37542024-01-24T19:07:03.239Z 17 Acked 1016 Read 1 Done Done Sent false
37552024-01-24T19:07:03.239Z 18 Acked 1017 Write 1 Done Done Sent false
37562024-01-24T19:07:03.239Z 19 Acked 1018 Read 1 Done Done Sent false
37572024-01-24T19:07:03.239Z 20 Acked 1019 Read 3 Done Done Sent false
37582024-01-24T19:07:03.242Z 21 Acked 1020 Write 7 Done Done Sent false
37592024-01-24T19:07:03.242Z 22 Acked 1021 Read 5 Done Done Sent false
37602024-01-24T19:07:03.242Z 23 Acked 1022 Write 8 Done Done Sent false
37612024-01-24T19:07:03.242Z 24 Acked 1023 Flush 0 Done Done Sent false
37622024-01-24T19:07:03.242Z 25 Acked 1024 Write 10 Done Done Sent false
37632024-01-24T19:07:03.242Z 26 Acked 1025 Read 6 Done Done Sent false
37642024-01-24T19:07:03.242Z 27 Acked 1026 Write 2 Done Done Sent false
37652024-01-24T19:07:03.242Z 28 Acked 1027 Read 10 Done Done Sent false
37662024-01-24T19:07:03.243Z 29 Acked 1028 Read 2 Done Done Sent false
37672024-01-24T19:07:03.243Z 30 Acked 1029 Write 7 Sent Sent Sent false
37682024-01-24T19:07:03.243Z STATES DS:0 DS:1 DS:2 TOTAL
37692024-01-24T19:07:03.243Z New 0 0 0 0
37702024-01-24T19:07:03.243Z Sent 1 1 27 29
37712024-01-24T19:07:03.243Z Done 26 26 0 52
37722024-01-24T19:07:03.243Z Skipped 0 0 0 0
37732024-01-24T19:07:03.243Z Error 0 0 0 0
37742024-01-24T19:07:03.243ZLast Flush: 1023 1023 1002
37752024-01-24T19:07:03.243ZDownstairs last five completed: 1002 1001 1000
37762024-01-24T19:07:03.243ZUpstairs last five completed: 30 29 28 27 26
37772024-01-24T19:07:03.243ZWrote out file "/tmp/test_repair_verify.data"
37782024-01-24T19:07:03.243Z
37792024-01-24T19:07:03.243ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
37802024-01-24T19:07:03.243ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
37812024-01-24T19:07:03.243Z 0 000-019 34 34 23 55 55 33 F F F <---
37822024-01-24T19:07:03.243Z 1 020-039 24 24 24 34 34 34 F F F
37832024-01-24T19:07:03.244Z 2 040-059 26 26 26 40 40 40 F F F
37842024-01-24T19:07:03.244Z 3 060-079 30 30 30 48 48 48 F F F
37852024-01-24T19:07:03.244Z 4 080-099 30 30 30 48 48 48 F F F
37862024-01-24T19:07:03.244Z 5 100-119 34 34 34 53 53 53 F F F
37872024-01-24T19:07:03.244Z 6 120-139 34 34 34 55 55 53 F F F <---
37882024-01-24T19:07:03.244Z 7 140-159 28 28 28 44 44 44 F F F
37892024-01-24T19:07:03.244Z 8 160-179 33 33 33 52 52 52 F F F
37902024-01-24T19:07:03.244Z 9 180-199 30 30 30 47 47 47 F F F
37912024-01-24T19:07:03.244Z 10 200-219 30 30 30 47 47 47 F F F
37922024-01-24T19:07:03.244Z 11 220-239 26 26 26 42 42 42 F F F
37932024-01-24T19:07:03.244Z 12 240-259 27 27 27 43 43 43 F F F
37942024-01-24T19:07:03.244Z 13 260-279 26 26 26 40 40 40 F F F
37952024-01-24T19:07:03.245Z 14 280-299 33 33 33 52 52 52 F F F
37962024-01-24T19:07:03.245Z 15 300-319 29 29 29 46 46 46 F F F
37972024-01-24T19:07:03.245Z 16 320-339 34 34 33 55 55 52 T T F <---
37982024-01-24T19:07:03.245Z 17 340-359 34 34 31 55 55 49 F F F <---
37992024-01-24T19:07:03.245Z 18 360-379 33 33 33 52 52 52 F F F
38002024-01-24T19:07:03.245Z 19 380-399 28 28 28 45 45 45 F F F
38012024-01-24T19:07:03.245Z 20 400-419 34 34 30 54 54 48 T T F <---
38022024-01-24T19:07:03.245Z 21 420-439 30 30 30 47 47 47 T T F <---
38032024-01-24T19:07:03.245Z 22 440-459 33 33 33 52 52 52 F F F
38042024-01-24T19:07:03.245Z 23 460-479 34 34 26 55 55 40 F F F <---
38052024-01-24T19:07:03.245Z 24 480-499 34 34 30 55 55 48 F F F <---
38062024-01-24T19:07:03.245Z 25 500-519 32 32 32 51 51 51 F F F
38072024-01-24T19:07:03.269Z 26 520-539 32 32 32 51 51 51 F F F
38082024-01-24T19:07:03.269Z 27 540-559 32 32 32 50 50 50 F F F
38092024-01-24T19:07:03.270Z 28 560-579 32 32 32 50 50 50 F F F
38102024-01-24T19:07:03.270Z 29 580-599 34 34 32 55 55 50 F F F <---
38112024-01-24T19:07:03.270ZMax gen: 34, Max flush: 55
38122024-01-24T19:07:03.270ZError: Difference in extent metadata found!
38132024-01-24T19:07:03.270ZA Difference in extent metadata is expected here
38142024-01-24T19:07:03.270ZOn loop 17
38152024-01-24T19:07:03.270Z
38162024-01-24T19:07:03.270ZVerifying data now
38172024-01-24T19:07:08.049ZNo change after verify
38182024-01-24T19:07:08.050ZLoop: 17 Downstairs dump after verify (and repair):
38192024-01-24T19:07:08.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38202024-01-24T19:07:08.072Z 0 000-019 34 34 34 55 55 55 F F F
38212024-01-24T19:07:08.090Z 1 020-039 24 24 24 34 34 34 F F F
38222024-01-24T19:07:08.090Z 2 040-059 26 26 26 40 40 40 F F F
38232024-01-24T19:07:08.090Z 3 060-079 30 30 30 48 48 48 F F F
38242024-01-24T19:07:08.091Z 4 080-099 30 30 30 48 48 48 F F F
38252024-01-24T19:07:08.091Z 5 100-119 34 34 34 53 53 53 F F F
38262024-01-24T19:07:08.091Z 6 120-139 34 34 34 55 55 55 F F F
38272024-01-24T19:07:08.091Z 7 140-159 28 28 28 44 44 44 F F F
38282024-01-24T19:07:08.091Z 8 160-179 33 33 33 52 52 52 F F F
38292024-01-24T19:07:08.091Z 9 180-199 30 30 30 47 47 47 F F F
38302024-01-24T19:07:08.091Z 10 200-219 30 30 30 47 47 47 F F F
38312024-01-24T19:07:08.091Z 11 220-239 26 26 26 42 42 42 F F F
38322024-01-24T19:07:08.091Z 12 240-259 27 27 27 43 43 43 F F F
38332024-01-24T19:07:08.092Z 13 260-279 26 26 26 40 40 40 F F F
38342024-01-24T19:07:08.092Z 14 280-299 33 33 33 52 52 52 F F F
38352024-01-24T19:07:08.092Z 15 300-319 29 29 29 46 46 46 F F F
38362024-01-24T19:07:08.093Z 16 320-339 35 35 35 56 56 56 F F F
38372024-01-24T19:07:08.093Z 17 340-359 34 34 34 55 55 55 F F F
38382024-01-24T19:07:08.093Z 18 360-379 33 33 33 52 52 52 F F F
38392024-01-24T19:07:08.093Z 19 380-399 28 28 28 45 45 45 F F F
38402024-01-24T19:07:08.093Z 20 400-419 35 35 35 56 56 56 F F F
38412024-01-24T19:07:08.093Z 21 420-439 35 35 35 56 56 56 F F F
38422024-01-24T19:07:08.094Z 22 440-459 33 33 33 52 52 52 F F F
38432024-01-24T19:07:08.094Z 23 460-479 34 34 34 55 55 55 F F F
38442024-01-24T19:07:08.094Z 24 480-499 34 34 34 55 55 55 F F F
38452024-01-24T19:07:08.094Z 25 500-519 32 32 32 51 51 51 F F F
38462024-01-24T19:07:08.095Z 26 520-539 32 32 32 51 51 51 F F F
38472024-01-24T19:07:08.095Z 27 540-559 32 32 32 50 50 50 F F F
38482024-01-24T19:07:08.095Z 28 560-579 32 32 32 50 50 50 F F F
38492024-01-24T19:07:08.095Z 29 580-599 34 34 34 55 55 55 F F F
38502024-01-24T19:07:08.095ZMax gen: 35, Max flush: 56
38512024-01-24T19:07:08.095Z
38522024-01-24T19:07:08.096ZBegin loop 18
38532024-01-24T19:07:08.096ZDownstairs to restart: 2
38542024-01-24T19:07:08.113Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.761801825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273}
38552024-01-24T19:07:08.114Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762283593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273}
38562024-01-24T19:07:08.115Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762351098Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273}
38572024-01-24T19:07:08.115Z{"msg":"Crucible 0f642a9b-1bd0-49ee-873d-095ab90812d3 has session id: ae52cdb9-2947-48bb-a0f8-5e584a5c05ff","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.76237748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38582024-01-24T19:07:08.116Z{"msg":"Upstairs opts: Upstairs UUID: 0f642a9b-1bd0-49ee-873d-095ab90812d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
38592024-01-24T19:07:08.116Z,"time":"2024-01-24T19:07:06.762452728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38602024-01-24T19:07:08.116Z{"msg":"Crucible stats registered with UUID: 0f642a9b-1bd0-49ee-873d-095ab90812d3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762583028Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38612024-01-24T19:07:08.116Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762692944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273}
38622024-01-24T19:07:08.116Z{"msg":"0f642a9b-1bd0-49ee-873d-095ab90812d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762851048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38632024-01-24T19:07:08.116Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762910942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38642024-01-24T19:07:08.117Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762939188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38652024-01-24T19:07:08.117Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.76296166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38662024-01-24T19:07:08.117Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.762984432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38672024-01-24T19:07:08.118Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763060774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38682024-01-24T19:07:08.118Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763108425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38692024-01-24T19:07:08.118Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.76313423Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38702024-01-24T19:07:08.118Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763284873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38712024-01-24T19:07:08.118Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763563345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38722024-01-24T19:07:08.118Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763600797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38732024-01-24T19:07:08.118Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.76362589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38742024-01-24T19:07:08.118Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.763661375Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38752024-01-24T19:07:08.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.764693521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38762024-01-24T19:07:08.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.764859227Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38772024-01-24T19:07:08.258Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.765030809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38782024-01-24T19:07:08.258Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.765078229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38792024-01-24T19:07:08.258Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.765360502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38802024-01-24T19:07:08.258Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.765418623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38812024-01-24T19:07:08.258Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.766101747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38822024-01-24T19:07:08.258Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.766186308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38832024-01-24T19:07:08.259Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.766325482Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38842024-01-24T19:07:08.259Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767087227Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38852024-01-24T19:07:08.259Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767226075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38862024-01-24T19:07:08.263Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767272734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38872024-01-24T19:07:08.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767329735Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38882024-01-24T19:07:08.263Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767373968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38892024-01-24T19:07:08.263Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767463587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38902024-01-24T19:07:08.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767503698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38912024-01-24T19:07:08.264Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767551791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38922024-01-24T19:07:08.264Z{"msg":"[0]R flush_numbers[0..12]: [55, 34, 40, 48, 48, 53, 55, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767579355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38932024-01-24T19:07:08.264Z{"msg":"[0]R generation[0..12]: [34, 24, 26, 30, 30, 34, 34, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.76760314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38942024-01-24T19:07:08.264Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767628273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38952024-01-24T19:07:08.264Z{"msg":"[1]R flush_numbers[0..12]: [55, 34, 40, 48, 48, 53, 55, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767652051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38962024-01-24T19:07:08.280Z{"msg":"[1]R generation[0..12]: [34, 24, 26, 30, 30, 34, 34, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
38972024-01-24T19:07:08.280Z,"time":"2024-01-24T19:07:06.767695851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38982024-01-24T19:07:08.280Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767723637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
38992024-01-24T19:07:08.280Z{"msg":"[2]R flush_numbers[0..12]: [55, 34, 40, 48, 48, 53, 55, 44, 52, 47, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767818879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39002024-01-24T19:07:08.280Z{"msg":"[2]R generation[0..12]: [34, 24, 26, 30, 30, 34, 34, 28, 33, 30, 30, 26]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767845244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39012024-01-24T19:07:08.280Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767900736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39022024-01-24T19:07:08.281Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767926471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39032024-01-24T19:07:08.281Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767949309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39042024-01-24T19:07:08.281Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767971001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39052024-01-24T19:07:08.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.767993111Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39062024-01-24T19:07:08.281Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768109214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39072024-01-24T19:07:08.281Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768186506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39082024-01-24T19:07:08.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768265622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39092024-01-24T19:07:08.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768342388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39102024-01-24T19:07:08.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768421424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39112024-01-24T19:07:08.282Z,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39122024-01-24T19:07:08.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768547549Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39132024-01-24T19:07:08.282Z{"msg":"0f642a9b-1bd0-49ee-873d-095ab90812d3 is now active with session: ae52cdb9-2947-48bb-a0f8-5e584a5c05ff","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768602599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39142024-01-24T19:07:08.282Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768626673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273,"session_id":"ae52cdb9-2947-48bb-a0f8-5e584a5c05ff"}
39152024-01-24T19:07:08.283Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:06.768669143Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1273}
39162024-01-24T19:07:08.283ZLoading write count information from file "/tmp/test_repair_verify.data"
39172024-01-24T19:07:08.283ZRun Repair workload
39182024-01-24T19:07:08.284Z01/30 Write block 91 len 36864 data: 2 2 4 4 3 4 5 4 4
39192024-01-24T19:07:08.284Z02/30 Read block 169 len 4096
39202024-01-24T19:07:08.284Z03/30 Read block 590 len 24576
39212024-01-24T19:07:08.284Z04/30 Write block 316 len 40960 data: 5 6 6 6 6 6 6 7 6 8
39222024-01-24T19:07:08.285Z05/30 Write block 94 len 20480 data: 5 4 5 6 5
39232024-01-24T19:07:08.285Z06/30 Read block 203 len 12288
39242024-01-24T19:07:08.285Z07/30 Flush
39252024-01-24T19:07:08.285Z08/30 Write block 43 len 32768 data: 4 5 6 4 3 3 3 3
39262024-01-24T19:07:08.287Z09/30 Read block 513 len 12288
39272024-01-24T19:07:08.287Z10/30 Flush
39282024-01-24T19:07:08.303Z11/30 Write block 308 len 32768 data: 2 2 2 3 3 3 3 5
39292024-01-24T19:07:08.304Z12/30 Read block 109 len 32768
39302024-01-24T19:07:08.304Z13/30 Write block 81 len 36864 data: 6 4 5 4 5 5 5 4 3
39312024-01-24T19:07:08.304Z14/30 Read block 292 len 20480
39322024-01-24T19:07:08.304Z15/30 Write block 119 len 16384 data: 5 5 6 6
39332024-01-24T19:07:08.304Z16/30 Flush
39342024-01-24T19:07:08.304Z17/30 Write block 193 len 8192 data: 4 4
39352024-01-24T19:07:08.306Z18/30 Write block 27 len 4096 data: 6
39362024-01-24T19:07:08.306Z19/30 Read block 388 len 20480
39372024-01-24T19:07:08.306Z20/30 Read block 25 len 8192
39382024-01-24T19:07:08.306Z21/30 Read block 55 len 16384
39392024-01-24T19:07:08.306Z22/30 Write block 452 len 24576 data: 5 6 6 5 6 6
39402024-01-24T19:07:08.306Z23/30 Write block 224 len 12288 data: 4 5 5
39412024-01-24T19:07:08.307Z24/30 Read block 387 len 16384
39422024-01-24T19:07:08.307Z25/30 Read block 397 len 20480
39432024-01-24T19:07:08.307Z26/30 Read block 199 len 32768
39442024-01-24T19:07:08.307Z27/30 Read block 46 len 12288
39452024-01-24T19:07:08.307Z28/30 Read block 344 len 8192
39462024-01-24T19:07:08.307Z29/30 Write block 254 len 28672 data: 2 2 2 2 4 4 4
39472024-01-24T19:07:08.308Z30/30 Read block 229 len 28672
39482024-01-24T19:07:08.308Z----------------------------------------------------------------
39492024-01-24T19:07:08.308Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39502024-01-24T19:07:08.308ZStates: Active Active Active
39512024-01-24T19:07:08.308ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39522024-01-24T19:07:08.308Z 1 Acked 1000 Write 9 Done Done Done false
39532024-01-24T19:07:08.309Z 2 Acked 1001 Read 1 Done Done Done false
39542024-01-24T19:07:08.309Z 3 Acked 1002 Read 6 Done Done Sent false
39552024-01-24T19:07:08.309Z 4 Acked 1003 Write 10 Done Done Sent false
39562024-01-24T19:07:08.309Z 5 Acked 1004 Write 5 Done Done Sent false
39572024-01-24T19:07:08.309Z 6 Acked 1005 Read 3 Done Done Sent false
39582024-01-24T19:07:08.309Z 7 Acked 1006 Flush 0 Done Done Sent false
39592024-01-24T19:07:08.309Z 8 Acked 1007 Write 8 Done Done Sent false
39602024-01-24T19:07:08.309Z 9 Acked 1008 Read 3 Done Done Sent false
39612024-01-24T19:07:08.310Z 10 Acked 1009 Flush 0 Done Done Sent false
39622024-01-24T19:07:08.310Z 11 Acked 1010 Write 8 Done Done Sent false
39632024-01-24T19:07:08.310Z 12 Acked 1011 Read 8 Done Done Sent false
39642024-01-24T19:07:08.310Z 13 Acked 1012 Write 9 Done Done Sent false
39652024-01-24T19:07:08.310Z 14 Acked 1013 Read 5 Done Done Sent false
39662024-01-24T19:07:08.310Z 15 Acked 1014 Write 4 Done Done Sent false
39672024-01-24T19:07:08.311Z 16 Acked 1015 Flush 0 Done Done Sent false
39682024-01-24T19:07:08.311Z 17 Acked 1016 Write 2 Done Done Sent false
39692024-01-24T19:07:08.311Z 18 Acked 1017 Write 1 Done Done Sent false
39702024-01-24T19:07:08.311Z 19 Acked 1018 Read 5 Done Done Sent false
39712024-01-24T19:07:08.311Z 20 Acked 1019 Read 2 Done Done Sent false
39722024-01-24T19:07:08.311Z 21 Acked 1020 Read 4 Done Done Sent false
39732024-01-24T19:07:08.312Z 22 Acked 1021 Write 6 Done Done Sent false
39742024-01-24T19:07:08.312Z 23 Acked 1022 Write 3 Done Done Sent false
39752024-01-24T19:07:08.312Z 24 Acked 1023 Read 4 Done Done Sent false
39762024-01-24T19:07:08.312Z 25 Acked 1024 Read 5 Done Done Sent false
39772024-01-24T19:07:08.312Z 26 Acked 1025 Read 8 Done Done Sent false
39782024-01-24T19:07:08.312Z 27 Acked 1026 Read 3 Done Done Sent false
39792024-01-24T19:07:08.312Z 28 Acked 1027 Read 2 Done Done Sent false
39802024-01-24T19:07:08.312Z 29 Acked 1028 Write 7 Done Done Sent false
39812024-01-24T19:07:08.313Z 30 Acked 1029 Read 7 Sent Done Sent false
39822024-01-24T19:07:08.313Z STATES DS:0 DS:1 DS:2 TOTAL
39832024-01-24T19:07:08.313Z New 0 0 0 0
39842024-01-24T19:07:08.313Z Sent 1 0 28 29
39852024-01-24T19:07:08.313Z Done 29 30 2 61
39862024-01-24T19:07:08.313Z Skipped 0 0 0 0
39872024-01-24T19:07:08.314Z Error 0 0 0 0
39882024-01-24T19:07:08.314ZLast Flush: 1015 1015 0
39892024-01-24T19:07:08.314ZDownstairs last five completed:
39902024-01-24T19:07:08.314ZUpstairs last five completed: 30 29 28 27 26
39912024-01-24T19:07:08.314ZWrote out file "/tmp/test_repair_verify.data"
39922024-01-24T19:07:08.314Z
39932024-01-24T19:07:08.314ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
39942024-01-24T19:07:08.315ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39952024-01-24T19:07:08.315Z 0 000-019 34 34 34 55 55 55 F F F
39962024-01-24T19:07:08.315Z 1 020-039 24 24 24 34 34 34 T T F <---
39972024-01-24T19:07:08.315Z 2 040-059 36 36 26 58 58 40 F F F <---
39982024-01-24T19:07:08.315Z 3 060-079 30 30 30 48 48 48 F F F
39992024-01-24T19:07:08.315Z 4 080-099 36 36 30 59 59 48 F F T <---
40002024-01-24T19:07:08.315Z 5 100-119 36 36 34 59 59 53 F F F <---
40012024-01-24T19:07:08.316Z 6 120-139 36 36 34 59 59 55 F F F <---
40022024-01-24T19:07:08.316Z 7 140-159 28 28 28 44 44 44 F F F
40032024-01-24T19:07:08.316Z 8 160-179 33 33 33 52 52 52 F F F
40042024-01-24T19:07:08.316Z 9 180-199 30 30 30 47 47 47 T T F <---
40052024-01-24T19:07:08.316Z 10 200-219 30 30 30 47 47 47 F F F
40062024-01-24T19:07:08.316Z 11 220-239 26 26 26 42 42 42 T T F <---
40072024-01-24T19:07:08.317Z 12 240-259 27 27 27 43 43 43 T T F <---
40082024-01-24T19:07:08.350Z 13 260-279 26 26 26 40 40 40 T T F <---
40092024-01-24T19:07:08.351Z 14 280-299 33 33 33 52 52 52 F F F
40102024-01-24T19:07:08.351Z 15 300-319 36 36 29 59 59 46 F F F <---
40112024-01-24T19:07:08.351Z 16 320-339 36 36 35 57 57 56 F F F <---
40122024-01-24T19:07:08.351Z 17 340-359 34 34 34 55 55 55 F F F
40132024-01-24T19:07:08.352Z 18 360-379 33 33 33 52 52 52 F F F
40142024-01-24T19:07:08.352Z 19 380-399 28 28 28 45 45 45 F F F
40152024-01-24T19:07:08.352Z 20 400-419 35 35 35 56 56 56 F F F
40162024-01-24T19:07:08.352Z 21 420-439 35 35 35 56 56 56 F F F
40172024-01-24T19:07:08.352Z 22 440-459 33 33 33 52 52 52 T T F <---
40182024-01-24T19:07:08.352Z 23 460-479 34 34 34 55 55 55 F F F
40192024-01-24T19:07:08.352Z 24 480-499 34 34 34 55 55 55 F F F
40202024-01-24T19:07:08.353Z 25 500-519 32 32 32 51 51 51 F F F
40212024-01-24T19:07:08.353Z 26 520-539 32 32 32 51 51 51 F F F
40222024-01-24T19:07:08.353Z 27 540-559 32 32 32 50 50 50 F F F
40232024-01-24T19:07:08.353Z 28 560-579 32 32 32 50 50 50 F F F
40242024-01-24T19:07:08.353Z 29 580-599 34 34 34 55 55 55 F F F
40252024-01-24T19:07:08.354ZMax gen: 36, Max flush: 59
40262024-01-24T19:07:08.354ZError: Difference in extent metadata found!
40272024-01-24T19:07:08.354ZA Difference in extent metadata is expected here
40282024-01-24T19:07:08.364ZOn loop 18
40292024-01-24T19:07:08.364Z
40302024-01-24T19:07:08.365ZVerifying data now
40312024-01-24T19:07:13.299ZNo change after verify
40322024-01-24T19:07:13.299ZLoop: 18 Downstairs dump after verify (and repair):
40332024-01-24T19:07:13.322ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40342024-01-24T19:07:13.323Z 0 000-019 34 34 34 55 55 55 F F F
40352024-01-24T19:07:13.332Z 1 020-039 37 37 37 60 60 60 F F F
40362024-01-24T19:07:13.332Z 2 040-059 36 36 36 58 58 58 F F F
40372024-01-24T19:07:13.332Z 3 060-079 30 30 30 48 48 48 F F F
40382024-01-24T19:07:13.332Z 4 080-099 36 36 36 59 59 59 F F F
40392024-01-24T19:07:13.332Z 5 100-119 36 36 36 59 59 59 F F F
40402024-01-24T19:07:13.333Z 6 120-139 36 36 36 59 59 59 F F F
40412024-01-24T19:07:13.333Z 7 140-159 28 28 28 44 44 44 F F F
40422024-01-24T19:07:13.333Z 8 160-179 33 33 33 52 52 52 F F F
40432024-01-24T19:07:13.333Z 9 180-199 37 37 37 60 60 60 F F F
40442024-01-24T19:07:13.333Z 10 200-219 30 30 30 47 47 47 F F F
40452024-01-24T19:07:13.333Z 11 220-239 37 37 37 60 60 60 F F F
40462024-01-24T19:07:13.333Z 12 240-259 37 37 37 60 60 60 F F F
40472024-01-24T19:07:13.333Z 13 260-279 37 37 37 60 60 60 F F F
40482024-01-24T19:07:13.334Z 14 280-299 33 33 33 52 52 52 F F F
40492024-01-24T19:07:13.334Z 15 300-319 36 36 36 59 59 59 F F F
40502024-01-24T19:07:13.334Z 16 320-339 36 36 36 57 57 57 F F F
40512024-01-24T19:07:13.334Z 17 340-359 34 34 34 55 55 55 F F F
40522024-01-24T19:07:13.334Z 18 360-379 33 33 33 52 52 52 F F F
40532024-01-24T19:07:13.334Z 19 380-399 28 28 28 45 45 45 F F F
40542024-01-24T19:07:13.334Z 20 400-419 35 35 35 56 56 56 F F F
40552024-01-24T19:07:13.334Z 21 420-439 35 35 35 56 56 56 F F F
40562024-01-24T19:07:13.335Z 22 440-459 37 37 37 60 60 60 F F F
40572024-01-24T19:07:13.335Z 23 460-479 34 34 34 55 55 55 F F F
40582024-01-24T19:07:13.335Z 24 480-499 34 34 34 55 55 55 F F F
40592024-01-24T19:07:13.335Z 25 500-519 32 32 32 51 51 51 F F F
40602024-01-24T19:07:13.335Z 26 520-539 32 32 32 51 51 51 F F F
40612024-01-24T19:07:13.335Z 27 540-559 32 32 32 50 50 50 F F F
40622024-01-24T19:07:13.335Z 28 560-579 32 32 32 50 50 50 F F F
40632024-01-24T19:07:13.335Z 29 580-599 34 34 34 55 55 55 F F F
40642024-01-24T19:07:13.335ZMax gen: 37, Max flush: 60
40652024-01-24T19:07:13.336Z
40662024-01-24T19:07:13.336ZBegin loop 19
40672024-01-24T19:07:13.336ZDownstairs to restart: 2
40682024-01-24T19:07:13.355Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.012311362Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281}
40692024-01-24T19:07:13.355Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.012774346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281}
40702024-01-24T19:07:13.375Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.012804199Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281}
40712024-01-24T19:07:13.375Z{"msg":"Crucible cd433c9f-0eb5-4517-be7a-7cef7f9f9db1 has session id: 0a63f08a-3b6a-4add-9943-dd280c631f9f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.012872226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40722024-01-24T19:07:13.376Z{"msg":"Upstairs opts: Upstairs UUID: cd433c9f-0eb5-4517-be7a-7cef7f9f9db1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.012933778Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40732024-01-24T19:07:13.376Z{"msg":"Crucible stats registered with UUID: cd433c9f-0eb5-4517-be7a-7cef7f9f9db1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.01306017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40742024-01-24T19:07:13.378ZCrucible runtime is spawned
40752024-01-24T19:07:13.378Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013277035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281}
40762024-01-24T19:07:13.378Z{"msg":"cd433c9f-0eb5-4517-be7a-7cef7f9f9db1 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.0134608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40772024-01-24T19:07:13.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013541706Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40782024-01-24T19:07:13.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013655469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40792024-01-24T19:07:13.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013700488Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40802024-01-24T19:07:13.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013856811Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40812024-01-24T19:07:13.378Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.013920235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40822024-01-24T19:07:13.378Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.01409368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40832024-01-24T19:07:13.379Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014321978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40842024-01-24T19:07:13.379Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014427447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40852024-01-24T19:07:13.380Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014511143Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40862024-01-24T19:07:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014561922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40872024-01-24T19:07:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014642373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40882024-01-24T19:07:13.380Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.014686536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40892024-01-24T19:07:13.380Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.015323653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40902024-01-24T19:07:13.380Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.015397316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40912024-01-24T19:07:13.380Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.015426211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40922024-01-24T19:07:13.380Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.015457298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40932024-01-24T19:07:13.380Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.016411785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40942024-01-24T19:07:13.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.016478552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40952024-01-24T19:07:13.383Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.016506119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40962024-01-24T19:07:13.384Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.016553361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40972024-01-24T19:07:13.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017441566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40982024-01-24T19:07:13.384Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017550255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
40992024-01-24T19:07:13.384Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.01757823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41002024-01-24T19:07:13.384Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017602311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41012024-01-24T19:07:13.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017652009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41022024-01-24T19:07:13.384Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017677579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41032024-01-24T19:07:13.384Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.017721533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41042024-01-24T19:07:13.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018403808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41052024-01-24T19:07:13.393Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018490515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41062024-01-24T19:07:13.393Z{"msg":"[0]R flush_numbers[0..12]: [55, 60, 58, 48, 59, 59, 59, 44, 52, 60, 47, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018584925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41072024-01-24T19:07:13.393Z{"msg":"[0]R generation[0..12]: [34, 37, 36, 30, 36, 36, 36, 28, 33, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018634566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41082024-01-24T19:07:13.393Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018668127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41092024-01-24T19:07:13.393Z{"msg":"[1]R flush_numbers[0..12]: [55, 60, 58, 48, 59, 59, 59, 44, 52, 60, 47, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018706152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
41102024-01-24T19:07:13.393Zsession_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41112024-01-24T19:07:13.393Z{"msg":"[1]R generation[0..12]: [34, 37, 36, 30, 36, 36, 36, 28, 33, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018749113Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41122024-01-24T19:07:13.393Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018822467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41132024-01-24T19:07:13.393Z{"msg":"[2]R flush_numbers[0..12]: [55, 60, 58, 48, 59, 59, 59, 44, 52, 60, 47, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018868926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41142024-01-24T19:07:13.394Z{"msg":"[2]R generation[0..12]: [34, 37, 36, 30, 36, 36, 36, 28, 33, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018925613Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41152024-01-24T19:07:13.394Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.018972105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41162024-01-24T19:07:13.394Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019043154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41172024-01-24T19:07:13.394Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019087593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41182024-01-24T19:07:13.394Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019149336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41192024-01-24T19:07:13.394Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019192089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41202024-01-24T19:07:13.394Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019284184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41212024-01-24T19:07:13.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019323698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41222024-01-24T19:07:13.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019352987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41232024-01-24T19:07:13.394Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019409422Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41242024-01-24T19:07:13.395Z":1281,"client":"2","":"downstairs","session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41252024-01-24T19:07:13.395Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019440524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41262024-01-24T19:07:13.395Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.01946157Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41272024-01-24T19:07:13.396Z{"msg":"cd433c9f-0eb5-4517-be7a-7cef7f9f9db1 is now active with session: 0a63f08a-3b6a-4add-9943-dd280c631f9f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019484207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41282024-01-24T19:07:13.397Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019504928Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281,"session_id":"0a63f08a-3b6a-4add-9943-dd280c631f9f"}
41292024-01-24T19:07:13.397Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:12.019529678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1281}
41302024-01-24T19:07:13.398ZLoading write count information from file "/tmp/test_repair_verify.data"
41312024-01-24T19:07:13.398ZRun Repair workload
41322024-01-24T19:07:13.398Z01/30 Write block 126 len 28672 data: 5 4 3 4 4 3 3
41332024-01-24T19:07:13.398Z02/30 Read block 468 len 4096
41342024-01-24T19:07:13.398Z03/30 Write block 478 len 36864 data: 4 4 5 6 6 4 4 4 4
41352024-01-24T19:07:13.398Z04/30 Flush
41362024-01-24T19:07:13.405Z05/30 Write block 293 len 4096 data: 5
41372024-01-24T19:07:13.405Z06/30 Write block 17 len 8192 data: 3 2
41382024-01-24T19:07:13.405Z07/30 Read block 542 len 28672
41392024-01-24T19:07:13.406Z08/30 Flush
41402024-01-24T19:07:13.406Z09/30 Write block 507 len 4096 data: 4
41412024-01-24T19:07:13.406Z10/30 Read block 456 len 12288
41422024-01-24T19:07:13.406Z11/30 Read block 204 len 12288
41432024-01-24T19:07:13.406Z12/30 Read block 171 len 16384
41442024-01-24T19:07:13.407Z13/30 Read block 309 len 32768
41452024-01-24T19:07:13.408Z14/30 Flush
41462024-01-24T19:07:13.408Z15/30 Write block 79 len 24576 data: 5 5 7 5 6 5
41472024-01-24T19:07:13.408Z16/30 Read block 202 len 40960
41482024-01-24T19:07:13.408Z17/30 Read block 361 len 8192
41492024-01-24T19:07:13.408Z18/30 Flush
41502024-01-24T19:07:13.408Z19/30 Write block 55 len 40960 data: 3 4 4 5 5 4 3 2 3 4
41512024-01-24T19:07:13.408Z20/30 Read block 247 len 32768
41522024-01-24T19:07:13.409Z21/30 Write block 209 len 8192 data: 3 3
41532024-01-24T19:07:13.409Z22/30 Write block 437 len 36864 data: 3 2 2 2 2 3 4 4 4
41542024-01-24T19:07:13.409Z23/30 Read block 69 len 16384
41552024-01-24T19:07:13.409Z24/30 Read block 340 len 24576
41562024-01-24T19:07:13.409Z25/30 Write block 246 len 12288 data: 6 4 3
41572024-01-24T19:07:13.410Z26/30 Write block 30 len 8192 data: 6 7
41582024-01-24T19:07:13.423Z27/30 Write block 307 len 8192 data: 3 3
41592024-01-24T19:07:13.423Z28/30 Read block 248 len 24576
41602024-01-24T19:07:13.423Z29/30 Read block 407 len 4096
41612024-01-24T19:07:13.424Z30/30 Write block 72 len 8192 data: 3 3
41622024-01-24T19:07:13.424Z----------------------------------------------------------------
41632024-01-24T19:07:13.424Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:26
41642024-01-24T19:07:13.424ZStates: Active Active Active
41652024-01-24T19:07:13.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41662024-01-24T19:07:13.424Z 5 Acked 1004 Write 1 Done Done Sent false
41672024-01-24T19:07:13.425Z 6 Acked 1005 Write 2 Done Done Sent false
41682024-01-24T19:07:13.426Z 7 Acked 1006 Read 7 Done Done Sent false
41692024-01-24T19:07:13.426Z 8 Acked 1007 Flush 0 Done Done Sent false
41702024-01-24T19:07:13.427Z 9 Acked 1008 Write 1 Done Done Sent false
41712024-01-24T19:07:13.427Z 10 Acked 1009 Read 3 Done Done Sent false
41722024-01-24T19:07:13.427Z 11 Acked 1010 Read 3 Done Done Sent false
41732024-01-24T19:07:13.427Z 12 Acked 1011 Read 4 Done Done Sent false
41742024-01-24T19:07:13.427Z 13 Acked 1012 Read 8 Done Done Sent false
41752024-01-24T19:07:13.427Z 14 Acked 1013 Flush 0 Done Done Sent false
41762024-01-24T19:07:13.427Z 15 Acked 1014 Write 6 Done Done Sent false
41772024-01-24T19:07:13.427Z 16 Acked 1015 Read 10 Done Done Sent false
41782024-01-24T19:07:13.427Z 17 Acked 1016 Read 2 Done Done Sent false
41792024-01-24T19:07:13.427Z 18 Acked 1017 Flush 0 Done Done Sent false
41802024-01-24T19:07:13.428Z 19 Acked 1018 Write 10 Done Done Sent false
41812024-01-24T19:07:13.428Z 20 Acked 1019 Read 8 Done Done Sent false
41822024-01-24T19:07:13.428Z 21 Acked 1020 Write 2 Done Done Sent false
41832024-01-24T19:07:13.428Z 22 Acked 1021 Write 9 Done Done Sent false
41842024-01-24T19:07:13.428Z 23 Acked 1022 Read 4 Done Done Sent false
41852024-01-24T19:07:13.428Z 24 Acked 1023 Read 6 Done Done Sent false
41862024-01-24T19:07:13.429Z 25 Acked 1024 Write 3 Done Done Sent false
41872024-01-24T19:07:13.429Z 26 Acked 1025 Write 2 Done Done Sent false
41882024-01-24T19:07:13.429Z 27 Acked 1026 Write 2 Done Done Sent false
41892024-01-24T19:07:13.429Z 28 Acked 1027 Read 6 Done Done Sent false
41902024-01-24T19:07:13.429Z 29 Acked 1028 Read 1 Done Done Sent false
41912024-01-24T19:07:13.430Z 30 Acked 1029 Write 2 Sent Sent Sent false
41922024-01-24T19:07:13.430Z STATES DS:0 DS:1 DS:2 TOTAL
41932024-01-24T19:07:13.430Z New 0 0 0 0
41942024-01-24T19:07:13.430Z Sent 1 1 26 28
41952024-01-24T19:07:13.431Z Done 25 25 0 50
41962024-01-24T19:07:13.431Z Skipped 0 0 0 0
41972024-01-24T19:07:13.431Z Error 0 0 0 0
41982024-01-24T19:07:13.431ZLast Flush: 1017 1017 1003
41992024-01-24T19:07:13.432ZDownstairs last five completed: 1003 1002 1001 1000
42002024-01-24T19:07:13.432ZUpstairs last five completed: 30 29 28 27 26
42012024-01-24T19:07:13.432ZWrote out file "/tmp/test_repair_verify.data"
42022024-01-24T19:07:13.432Z
42032024-01-24T19:07:13.846ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42042024-01-24T19:07:13.864ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42052024-01-24T19:07:13.865Z 0 000-019 38 38 34 62 62 55 F F F <---
42062024-01-24T19:07:13.865Z 1 020-039 37 37 37 60 60 60 T T F <---
42072024-01-24T19:07:13.865Z 2 040-059 36 36 36 58 58 58 T T F <---
42082024-01-24T19:07:13.865Z 3 060-079 38 38 30 64 64 48 T T F <---
42092024-01-24T19:07:13.865Z 4 080-099 38 38 36 64 64 59 F F F <---
42102024-01-24T19:07:13.865Z 5 100-119 36 36 36 59 59 59 F F F
42112024-01-24T19:07:13.865Z 6 120-139 38 38 38 61 61 61 F F F
42122024-01-24T19:07:13.865Z 7 140-159 28 28 28 44 44 44 F F F
42132024-01-24T19:07:13.865Z 8 160-179 33 33 33 52 52 52 F F F
42142024-01-24T19:07:13.866Z 9 180-199 37 37 37 60 60 60 F F F
42152024-01-24T19:07:13.866Z 10 200-219 30 30 30 47 47 47 T T F <---
42162024-01-24T19:07:13.866Z 11 220-239 37 37 37 60 60 60 F F F
42172024-01-24T19:07:13.866Z 12 240-259 37 37 37 60 60 60 T T F <---
42182024-01-24T19:07:13.866Z 13 260-279 37 37 37 60 60 60 F F F
42192024-01-24T19:07:13.866Z 14 280-299 38 38 33 62 62 52 F F F <---
42202024-01-24T19:07:13.866Z 15 300-319 36 36 36 59 59 59 T T F <---
42212024-01-24T19:07:13.866Z 16 320-339 36 36 36 57 57 57 F F F
42222024-01-24T19:07:13.867Z 17 340-359 34 34 34 55 55 55 F F F
42232024-01-24T19:07:13.867Z 18 360-379 33 33 33 52 52 52 F F F
42242024-01-24T19:07:13.867Z 19 380-399 28 28 28 45 45 45 F F F
42252024-01-24T19:07:13.867Z 20 400-419 35 35 35 56 56 56 F F F
42262024-01-24T19:07:13.867Z 21 420-439 35 35 35 56 56 56 T T F <---
42272024-01-24T19:07:13.867Z 22 440-459 37 37 37 60 60 60 T T F <---
42282024-01-24T19:07:13.867Z 23 460-479 38 38 38 61 61 61 F F F
42292024-01-24T19:07:13.868Z 24 480-499 38 38 38 61 61 61 F F F
42302024-01-24T19:07:13.868Z 25 500-519 38 38 32 63 63 51 F F F <---
42312024-01-24T19:07:13.868Z 26 520-539 32 32 32 51 51 51 F F F
42322024-01-24T19:07:13.869Z 27 540-559 32 32 32 50 50 50 F F F
42332024-01-24T19:07:13.869Z 28 560-579 32 32 32 50 50 50 F F F
42342024-01-24T19:07:13.869Z 29 580-599 34 34 34 55 55 55 F F F
42352024-01-24T19:07:13.869ZMax gen: 38, Max flush: 64
42362024-01-24T19:07:13.869ZError: Difference in extent metadata found!
42372024-01-24T19:07:13.869ZA Difference in extent metadata is expected here
42382024-01-24T19:07:13.869ZOn loop 19
42392024-01-24T19:07:13.869Z
42402024-01-24T19:07:13.870ZVerifying data now
42412024-01-24T19:07:18.574ZNo change after verify
42422024-01-24T19:07:18.575ZLoop: 19 Downstairs dump after verify (and repair):
42432024-01-24T19:07:18.602ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42442024-01-24T19:07:18.602Z 0 000-019 38 38 38 62 62 62 F F F
42452024-01-24T19:07:18.602Z 1 020-039 39 39 39 65 65 65 F F F
42462024-01-24T19:07:18.602Z 2 040-059 39 39 39 65 65 65 F F F
42472024-01-24T19:07:18.603Z 3 060-079 39 39 39 65 65 65 F F F
42482024-01-24T19:07:18.605Z 4 080-099 38 38 38 64 64 64 F F F
42492024-01-24T19:07:18.605Z 5 100-119 36 36 36 59 59 59 F F F
42502024-01-24T19:07:18.605Z 6 120-139 38 38 38 61 61 61 F F F
42512024-01-24T19:07:18.605Z 7 140-159 28 28 28 44 44 44 F F F
42522024-01-24T19:07:18.605Z 8 160-179 33 33 33 52 52 52 F F F
42532024-01-24T19:07:18.606Z 9 180-199 37 37 37 60 60 60 F F F
42542024-01-24T19:07:18.606Z 10 200-219 39 39 39 65 65 65 F F F
42552024-01-24T19:07:18.606Z 11 220-239 37 37 37 60 60 60 F F F
42562024-01-24T19:07:18.606Z 12 240-259 39 39 39 65 65 65 F F F
42572024-01-24T19:07:18.606Z 13 260-279 37 37 37 60 60 60 F F F
42582024-01-24T19:07:18.607Z 14 280-299 38 38 38 62 62 62 F F F
42592024-01-24T19:07:18.607Z 15 300-319 39 39 39 65 65 65 F F F
42602024-01-24T19:07:18.607Z 16 320-339 36 36 36 57 57 57 F F F
42612024-01-24T19:07:18.607Z 17 340-359 34 34 34 55 55 55 F F F
42622024-01-24T19:07:18.607Z 18 360-379 33 33 33 52 52 52 F F F
42632024-01-24T19:07:18.607Z 19 380-399 28 28 28 45 45 45 F F F
42642024-01-24T19:07:18.608Z 20 400-419 35 35 35 56 56 56 F F F
42652024-01-24T19:07:18.608Z 21 420-439 39 39 39 65 65 65 F F F
42662024-01-24T19:07:18.608Z 22 440-459 39 39 39 65 65 65 F F F
42672024-01-24T19:07:18.608Z 23 460-479 38 38 38 61 61 61 F F F
42682024-01-24T19:07:18.608Z 24 480-499 38 38 38 61 61 61 F F F
42692024-01-24T19:07:18.608Z 25 500-519 38 38 38 63 63 63 F F F
42702024-01-24T19:07:18.609Z 26 520-539 32 32 32 51 51 51 F F F
42712024-01-24T19:07:18.609Z 27 540-559 32 32 32 50 50 50 F F F
42722024-01-24T19:07:18.609Z 28 560-579 32 32 32 50 50 50 F F F
42732024-01-24T19:07:18.612Z 29 580-599 34 34 34 55 55 55 F F F
42742024-01-24T19:07:18.612ZMax gen: 39, Max flush: 65
42752024-01-24T19:07:18.612Z
42762024-01-24T19:07:18.612ZBegin loop 20
42772024-01-24T19:07:18.612ZDownstairs to restart: 1
42782024-01-24T19:07:19.045Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.260449814Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289}
42792024-01-24T19:07:19.046Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.260937061Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289}
42802024-01-24T19:07:19.046Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.260993993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289}
42812024-01-24T19:07:19.046Z{"msg":"Crucible 6d87af95-dfc4-4db0-bcf6-ac80103b8646 has session id: 896a606b-fe32-42c4-839a-cae7d9cf7cb1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261025276Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42822024-01-24T19:07:19.046Z{"msg":"Upstairs opts: Upstairs UUID: 6d87af95-dfc4-4db0-bcf6-ac80103b8646, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261083297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42832024-01-24T19:07:19.046Z{"msg":"Crucible stats registered with UUID: 6d87af95-dfc4-4db0-bcf6-ac80103b8646","v":0,"name":"crucible","level":30Crucible runtime is spawned
42842024-01-24T19:07:19.046Z,"time":"2024-01-24T19:07:17.261202449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42852024-01-24T19:07:19.046Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261451229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289}
42862024-01-24T19:07:19.062Z{"msg":"6d87af95-dfc4-4db0-bcf6-ac80103b8646 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26152838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42872024-01-24T19:07:19.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26159826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42882024-01-24T19:07:19.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.2616773Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42892024-01-24T19:07:19.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26175399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42902024-01-24T19:07:19.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261801011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42912024-01-24T19:07:19.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261865846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42922024-01-24T19:07:19.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261935479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42932024-01-24T19:07:19.064Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.261964396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42942024-01-24T19:07:19.064Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.262028353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42952024-01-24T19:07:19.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.262099389Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42962024-01-24T19:07:19.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.262145612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42972024-01-24T19:07:19.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.262188691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42982024-01-24T19:07:19.065Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26223291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
42992024-01-24T19:07:19.065Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263452851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43002024-01-24T19:07:19.065Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263538989Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43012024-01-24T19:07:19.065Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263777459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43022024-01-24T19:07:19.065Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263861723Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43032024-01-24T19:07:19.066Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263944064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43042024-01-24T19:07:19.066Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.263991608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43052024-01-24T19:07:19.066Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.264649202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43062024-01-24T19:07:19.067Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.264730737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43072024-01-24T19:07:19.067Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.264867321Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43082024-01-24T19:07:19.067Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265615254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43092024-01-24T19:07:19.067Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26567637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43102024-01-24T19:07:19.068Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265757893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43112024-01-24T19:07:19.068Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265802976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43122024-01-24T19:07:19.069Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265842514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43132024-01-24T19:07:19.069Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265910982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43142024-01-24T19:07:19.069Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.265977167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43152024-01-24T19:07:19.069Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266048001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43162024-01-24T19:07:19.070Z{"msg":"[0]R flush_numbers[0..12]: [62, 65, 65, 65, 64, 59, 61, 44, 52, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266077187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43172024-01-24T19:07:19.070Z{"msg":"[0]R generation[0..12]: [38, 39, 39, 39, 38, 36, 38, 28, 33, 37, 39, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266122921Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43182024-01-24T19:07:19.071Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266150021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43192024-01-24T19:07:19.072Z{"msg":"[1]R flush_numbers[0..12]: [62, 65, 65, 65, 64, 59, 61, 44, 52, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266194101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43202024-01-24T19:07:19.072Z{"msg":"[1]R generation[0..12]: [38, 39, 39, 39, 38, 36, 38, 28, 33, 37, 39, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266220357Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43212024-01-24T19:07:19.073Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266267293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43222024-01-24T19:07:19.073ZWait for a query_work_queue command to finish before sending IO
43232024-01-24T19:07:19.073Z{"msg":"[2]R flush_numbers[0..12]: [62, 65, 65, 65, 64, 59, 61, 44, 52, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266300673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43242024-01-24T19:07:19.073Z{"msg":"[2]R generation[0..12]: [38, 39, 39, 39, 38, 36, 38, 28, 33, 37, 39, 37]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266324434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43252024-01-24T19:07:19.073Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266368774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43262024-01-24T19:07:19.075Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266393842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43272024-01-24T19:07:19.075Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266437358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43282024-01-24T19:07:19.076Z{"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266461736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43292024-01-24T19:07:19.076Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266483629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43302024-01-24T19:07:19.077Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266526937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43312024-01-24T19:07:19.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266549795Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43322024-01-24T19:07:19.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266572915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43332024-01-24T19:07:19.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266622948Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43342024-01-24T19:07:19.078Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26664781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43352024-01-24T19:07:19.078Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266668859Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43362024-01-24T19:07:19.078Z{"msg":"6d87af95-dfc4-4db0-bcf6-ac80103b8646 is now active with session: 896a606b-fe32-42c4-839a-cae7d9cf7cb1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266690315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43372024-01-24T19:07:19.078Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.26670962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289,"session_id":"896a606b-fe32-42c4-839a-cae7d9cf7cb1"}
43382024-01-24T19:07:19.079Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:17.266759064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1289}
43392024-01-24T19:07:19.079ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43402024-01-24T19:07:19.079ZLoading write count information from file "/tmp/test_repair_verify.data"
43412024-01-24T19:07:19.079ZRun Repair workload
43422024-01-24T19:07:19.079Z01/30 Write block 366 len 24576 data: 5 5 5 5 5 5
43432024-01-24T19:07:19.079Z02/30 Read block 477 len 20480
43442024-01-24T19:07:19.079Z03/30 Write block 569 len 16384 data: 2 2 2 2
43452024-01-24T19:07:19.079Z04/30 Read block 249 len 12288
43462024-01-24T19:07:19.079Z05/30 Read block 177 len 20480
43472024-01-24T19:07:19.079Z06/30 Read block 498 len 20480
43482024-01-24T19:07:19.532Z07/30 Write block 467 len 36864 data: 4 4 4 3 3 5 3 3 4
43492024-01-24T19:07:19.533Z08/30 Read block 279 len 12288
43502024-01-24T19:07:19.533Z09/30 Read block 473 len 16384
43512024-01-24T19:07:19.533Z10/30 Read block 522 len 8192
43522024-01-24T19:07:19.533Z11/30 Read block 195 len 8192
43532024-01-24T19:07:19.533Z12/30 Write block 371 len 40960 data: 6 5 5 5 5 4 4 4 4 4
43542024-01-24T19:07:19.534Z13/30 Read block 374 len 32768
43552024-01-24T19:07:19.534Z14/30 Read block 379 len 40960
43562024-01-24T19:07:19.535Z15/30 Read block 451 len 32768
43572024-01-24T19:07:19.535Z16/30 Write block 50 len 28672 data: 4 3 3 3 3 4 5
43582024-01-24T19:07:19.535Z17/30 Write block 277 len 16384 data: 3 4 7 6
43592024-01-24T19:07:19.535Z18/30 Write block 513 len 12288 data: 6 5 5
43602024-01-24T19:07:19.535Z19/30 Write block 225 len 16384 data: 6 6 6 5
43612024-01-24T19:07:19.535Z20/30 Write block 574 len 32768 data: 2 3 3 2 3 5 5 5
43622024-01-24T19:07:19.535Z21/30 Flush
43632024-01-24T19:07:19.535Z22/30 Write block 126 len 32768 data: 6 5 4 5 5 4 4 2
43642024-01-24T19:07:19.535Z23/30 Write block 423 len 20480 data: 4 4 3 3 3
43652024-01-24T19:07:19.536Z24/30 Flush
43662024-01-24T19:07:19.536Z25/30 Write block 377 len 4096 data: 5
43672024-01-24T19:07:19.536Z26/30 Read block 311 len 20480
43682024-01-24T19:07:19.536Z27/30 Read block 474 len 16384
43692024-01-24T19:07:19.536Z28/30 Write block 245 len 20480 data: 5 7 5 4 3
43702024-01-24T19:07:19.536Z29/30 Write block 210 len 16384 data: 4 3 3 3
43712024-01-24T19:07:19.536Z30/30 Write block 149 len 36864 data: 5 5 6 6 6 6 4 3 3
43722024-01-24T19:07:19.536Z----------------------------------------------------------------
43732024-01-24T19:07:19.536Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
43742024-01-24T19:07:19.536ZStates: Active Active Active
43752024-01-24T19:07:19.537ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43762024-01-24T19:07:19.537Z 1 Acked 1000 Write 6 Done Sent Done false
43772024-01-24T19:07:19.537Z 2 Acked 1001 Read 5 Done Sent Done false
43782024-01-24T19:07:19.537Z 3 Acked 1002 Write 4 Done Sent Done false
43792024-01-24T19:07:19.537Z 4 Acked 1003 Read 3 Done Sent Done false
43802024-01-24T19:07:19.537Z 5 Acked 1004 Read 5 Done Sent Done false
43812024-01-24T19:07:19.537Z 6 Acked 1005 Read 5 Done Sent Done false
43822024-01-24T19:07:19.537Z 7 Acked 1006 Write 9 Done Sent Done false
43832024-01-24T19:07:19.537Z 8 Acked 1007 Read 3 Done Sent Done false
43842024-01-24T19:07:19.537Z 9 Acked 1008 Read 4 Done Sent Done false
43852024-01-24T19:07:19.537Z 10 Acked 1009 Read 2 Done Sent Done false
43862024-01-24T19:07:19.537Z 11 Acked 1010 Read 2 Done Sent Done false
43872024-01-24T19:07:19.537Z 12 Acked 1011 Write 10 Done Sent Done false
43882024-01-24T19:07:19.537Z 13 Acked 1012 Read 8 Done Sent Done false
43892024-01-24T19:07:19.537Z 14 Acked 1013 Read 10 Done Sent Done false
43902024-01-24T19:07:19.537Z 15 Acked 1014 Read 8 Done Sent Done false
43912024-01-24T19:07:19.537Z 16 Acked 1015 Write 7 Done Sent Done false
43922024-01-24T19:07:19.538Z 17 Acked 1016 Write 4 Done Sent Done false
43932024-01-24T19:07:19.538Z 18 Acked 1017 Write 3 Done Sent Done false
43942024-01-24T19:07:19.538Z 19 Acked 1018 Write 4 Done Sent Done false
43952024-01-24T19:07:19.538Z 20 Acked 1019 Write 8 Done Sent Done false
43962024-01-24T19:07:19.538Z 21 Acked 1020 Flush 0 Done Sent Done false
43972024-01-24T19:07:19.538Z 22 Acked 1021 Write 8 Done Sent Done false
43982024-01-24T19:07:19.538Z 23 Acked 1022 Write 5 Done Sent Done false
43992024-01-24T19:07:19.538Z 24 Acked 1023 Flush 0 Done Sent Done false
44002024-01-24T19:07:19.538Z 25 Acked 1024 Write 1 Done Sent Done false
44012024-01-24T19:07:19.538Z 26 Acked 1025 Read 5 Done Sent Done false
44022024-01-24T19:07:19.540Z 27 Acked 1026 Read 4 Done Sent Done false
44032024-01-24T19:07:19.540Z 28 Acked 1027 Write 5 Done Sent Done false
44042024-01-24T19:07:19.540Z 29 Acked 1028 Write 4 Done Sent Done false
44052024-01-24T19:07:19.540Z 30 Acked 1029 Write 9 Sent Sent Sent false
44062024-01-24T19:07:19.540Z STATES DS:0 DS:1 DS:2 TOTAL
44072024-01-24T19:07:19.540Z New 0 0 0 0
44082024-01-24T19:07:19.540Z Sent 1 30 1 32
44092024-01-24T19:07:19.540Z Done 29 0 29 58
44102024-01-24T19:07:19.540Z Skipped 0 0 0 0
44112024-01-24T19:07:19.541Z Error 0 0 0 0
44122024-01-24T19:07:19.541ZLast Flush: 1023 0 1023
44132024-01-24T19:07:19.541ZDownstairs last five completed:
44142024-01-24T19:07:19.541ZUpstairs last five completed: 30 29 28 27 26
44152024-01-24T19:07:19.541ZWrote out file "/tmp/test_repair_verify.data"
44162024-01-24T19:07:19.541Z
44172024-01-24T19:07:19.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44182024-01-24T19:07:19.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44192024-01-24T19:07:19.541Z 0 000-019 38 38 38 62 62 62 F F F
44202024-01-24T19:07:19.542Z 1 020-039 39 39 39 65 65 65 F F F
44212024-01-24T19:07:19.542Z 2 040-059 40 39 40 66 65 66 F F F <---
44222024-01-24T19:07:19.542Z 3 060-079 39 39 39 65 65 65 F F F
44232024-01-24T19:07:19.542Z 4 080-099 38 38 38 64 64 64 F F F
44242024-01-24T19:07:19.542Z 5 100-119 36 36 36 59 59 59 F F F
44252024-01-24T19:07:19.542Z 6 120-139 40 38 40 67 61 67 F F F <---
44262024-01-24T19:07:19.542Z 7 140-159 28 28 28 44 44 44 T F T <---
44272024-01-24T19:07:19.546Z 8 160-179 33 33 33 52 52 52 F F F
44282024-01-24T19:07:19.546Z 9 180-199 37 37 37 60 60 60 F F F
44292024-01-24T19:07:19.546Z 10 200-219 39 39 39 65 65 65 T F T <---
44302024-01-24T19:07:19.546Z 11 220-239 40 37 40 66 60 66 F F F <---
44312024-01-24T19:07:19.547Z 12 240-259 39 39 39 65 65 65 T F T <---
44322024-01-24T19:07:19.547Z 13 260-279 40 37 40 66 60 66 F F F <---
44332024-01-24T19:07:19.547Z 14 280-299 40 38 40 66 62 66 F F F <---
44342024-01-24T19:07:19.547Z 15 300-319 39 39 39 65 65 65 F F F
44352024-01-24T19:07:19.547Z 16 320-339 36 36 36 57 57 57 F F F
44362024-01-24T19:07:19.547Z 17 340-359 34 34 34 55 55 55 F F F
44372024-01-24T19:07:19.548Z 18 360-379 40 33 40 66 52 66 T F T <---
44382024-01-24T19:07:19.548Z 19 380-399 40 28 40 66 45 66 F F F <---
44392024-01-24T19:07:19.548Z 20 400-419 35 35 35 56 56 56 F F F
44402024-01-24T19:07:19.548Z 21 420-439 40 39 40 67 65 67 F F F <---
44412024-01-24T19:07:19.548Z 22 440-459 39 39 39 65 65 65 F F F
44422024-01-24T19:07:19.548Z 23 460-479 40 38 40 66 61 66 F F F <---
44432024-01-24T19:07:19.548Z 24 480-499 38 38 38 61 61 61 F F F
44442024-01-24T19:07:19.548Z 25 500-519 40 38 40 66 63 66 F F F <---
44452024-01-24T19:07:19.548Z 26 520-539 32 32 32 51 51 51 F F F
44462024-01-24T19:07:19.548Z 27 540-559 32 32 32 50 50 50 F F F
44472024-01-24T19:07:19.548Z 28 560-579 40 32 40 66 50 66 F F F <---
44482024-01-24T19:07:20.792Z 29 580-599 40 34 40 66 55 66 F F F <---
44492024-01-24T19:07:20.792ZMax gen: 40, Max flush: 67
44502024-01-24T19:07:20.793ZError: Difference in extent metadata found!
44512024-01-24T19:07:20.793ZA Difference in extent metadata is expected here
44522024-01-24T19:07:20.793ZOn loop 20
44532024-01-24T19:07:20.793Z
44542024-01-24T19:07:20.793ZVerifying data now
44552024-01-24T19:07:24.008ZNo change after verify
44562024-01-24T19:07:24.008ZLoop: 20 Downstairs dump after verify (and repair):
44572024-01-24T19:07:24.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44582024-01-24T19:07:24.027Z 0 000-019 38 38 38 62 62 62 F F F
44592024-01-24T19:07:24.041Z 1 020-039 39 39 39 65 65 65 F F F
44602024-01-24T19:07:24.042Z 2 040-059 40 40 40 66 66 66 F F F
44612024-01-24T19:07:24.042Z 3 060-079 39 39 39 65 65 65 F F F
44622024-01-24T19:07:24.042Z 4 080-099 38 38 38 64 64 64 F F F
44632024-01-24T19:07:24.042Z 5 100-119 36 36 36 59 59 59 F F F
44642024-01-24T19:07:24.042Z 6 120-139 40 40 40 67 67 67 F F F
44652024-01-24T19:07:24.042Z 7 140-159 41 41 41 68 68 68 F F F
44662024-01-24T19:07:24.042Z 8 160-179 33 33 33 52 52 52 F F F
44672024-01-24T19:07:24.042Z 9 180-199 37 37 37 60 60 60 F F F
44682024-01-24T19:07:24.042Z 10 200-219 41 41 41 68 68 68 F F F
44692024-01-24T19:07:24.043Z 11 220-239 40 40 40 66 66 66 F F F
44702024-01-24T19:07:24.043Z 12 240-259 41 41 41 68 68 68 F F F
44712024-01-24T19:07:24.043Z 13 260-279 40 40 40 66 66 66 F F F
44722024-01-24T19:07:24.043Z 14 280-299 40 40 40 66 66 66 F F F
44732024-01-24T19:07:24.043Z 15 300-319 39 39 39 65 65 65 F F F
44742024-01-24T19:07:24.043Z 16 320-339 36 36 36 57 57 57 F F F
44752024-01-24T19:07:24.043Z 17 340-359 34 34 34 55 55 55 F F F
44762024-01-24T19:07:24.043Z 18 360-379 41 41 41 68 68 68 F F F
44772024-01-24T19:07:24.046Z 19 380-399 40 40 40 66 66 66 F F F
44782024-01-24T19:07:24.046Z 20 400-419 35 35 35 56 56 56 F F F
44792024-01-24T19:07:24.046Z 21 420-439 40 40 40 67 67 67 F F F
44802024-01-24T19:07:24.047Z 22 440-459 39 39 39 65 65 65 F F F
44812024-01-24T19:07:24.047Z 23 460-479 40 40 40 66 66 66 F F F
44822024-01-24T19:07:24.047Z 24 480-499 38 38 38 61 61 61 F F F
44832024-01-24T19:07:24.047Z 25 500-519 40 40 40 66 66 66 F F F
44842024-01-24T19:07:24.047Z 26 520-539 32 32 32 51 51 51 F F F
44852024-01-24T19:07:24.047Z 27 540-559 32 32 32 50 50 50 F F F
44862024-01-24T19:07:24.047Z 28 560-579 40 40 40 66 66 66 F F F
44872024-01-24T19:07:24.047Z 29 580-599 40 40 40 66 66 66 F F F
44882024-01-24T19:07:24.047ZMax gen: 41, Max flush: 68
44892024-01-24T19:07:24.048Z
44902024-01-24T19:07:24.048ZBegin loop 21
44912024-01-24T19:07:24.048ZDownstairs to restart: 2
44922024-01-24T19:07:24.056Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.714699577Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297}
44932024-01-24T19:07:24.056Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715204662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297}
44942024-01-24T19:07:24.066Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.71525762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297}
44952024-01-24T19:07:24.066Z{"msg":"Crucible 7658889e-375b-4f6f-9375-9dbf1bfa7a33 has session id: 2f77e0af-7050-4d3f-b3aa-f62b74103731","v":0,"name":"crucible","level":30Crucible runtime is spawned
44962024-01-24T19:07:24.066Z,"time":"2024-01-24T19:07:22.715414089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
44972024-01-24T19:07:24.066Z{"msg":"Upstairs opts: Upstairs UUID: 7658889e-375b-4f6f-9375-9dbf1bfa7a33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715628982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
44982024-01-24T19:07:24.066Z{"msg":"Crucible stats registered with UUID: 7658889e-375b-4f6f-9375-9dbf1bfa7a33","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715708488Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
44992024-01-24T19:07:24.066Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715781478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297}
45002024-01-24T19:07:24.067Z{"msg":"7658889e-375b-4f6f-9375-9dbf1bfa7a33 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715848463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45012024-01-24T19:07:24.067Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.715918167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45022024-01-24T19:07:24.067Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.71596103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45032024-01-24T19:07:24.067Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716057206Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45042024-01-24T19:07:24.067Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716101839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45052024-01-24T19:07:24.067Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716179266Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45062024-01-24T19:07:24.067Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716223841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45072024-01-24T19:07:24.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716263095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45082024-01-24T19:07:24.068Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716515541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45092024-01-24T19:07:24.068Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716591775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45102024-01-24T19:07:24.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716620542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45112024-01-24T19:07:24.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.71672457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45122024-01-24T19:07:24.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.716769465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45132024-01-24T19:07:24.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.717951506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45142024-01-24T19:07:24.069Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.718048306Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45152024-01-24T19:07:24.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.718123179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45162024-01-24T19:07:24.069Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.718233995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45172024-01-24T19:07:24.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.719013819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45182024-01-24T19:07:24.069Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.719092162Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45192024-01-24T19:07:24.069Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.719236714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45202024-01-24T19:07:24.069Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.719342684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45212024-01-24T19:07:24.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720032559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45222024-01-24T19:07:24.070Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720085794Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45232024-01-24T19:07:24.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720112106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45242024-01-24T19:07:24.070Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720135267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45252024-01-24T19:07:24.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720196256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45262024-01-24T19:07:24.070Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720222041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45272024-01-24T19:07:24.071Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.720273466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45282024-01-24T19:07:24.084Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721022624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45292024-01-24T19:07:24.085Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721105577Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45302024-01-24T19:07:24.085Z{"msg":"[0]R flush_numbers[0..12]: [62, 65, 66, 65, 64, 59, 67, 68, 52, 60, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721173063Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45312024-01-24T19:07:24.085Z{"msg":"[0]R generation[0..12]: [38, 39, 40, 39, 38, 36, 40, 41, 33, 37, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721220049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45322024-01-24T19:07:24.090Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.72124777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45332024-01-24T19:07:24.090Z{"msg":"[1]R flush_numbers[0..12]: [62, 65, 66, 65, 64, 59, 67, 68, 52, 60, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721303386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45342024-01-24T19:07:24.091ZWait for a query_work_queue command to finish before sending IO
45352024-01-24T19:07:24.091Z{"msg":"[1]R generation[0..12]: [38, 39, 40, 39, 38, 36, 40, 41, 33, 37, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721335907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45362024-01-24T19:07:24.091Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721359678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45372024-01-24T19:07:24.092Z{"msg":"[2]R flush_numbers[0..12]: [62, 65, 66, 65, 64, 59, 67, 68, 52, 60, 68, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721407213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45382024-01-24T19:07:24.092Z{"msg":"[2]R generation[0..12]: [38, 39, 40, 39, 38, 36, 40, 41, 33, 37, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721432917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45392024-01-24T19:07:24.092Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721476451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45402024-01-24T19:07:24.100Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721522526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45412024-01-24T19:07:24.100Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721547719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45422024-01-24T19:07:24.101Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721604396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45432024-01-24T19:07:24.101Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721629062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45442024-01-24T19:07:24.101Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721651198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45452024-01-24T19:07:24.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721670697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45462024-01-24T19:07:24.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721691927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45472024-01-24T19:07:24.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.72171329Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45482024-01-24T19:07:24.104Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721745037Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45492024-01-24T19:07:24.104Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721777783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45502024-01-24T19:07:24.105Z{"msg":"7658889e-375b-4f6f-9375-9dbf1bfa7a33 is now active with session: 2f77e0af-7050-4d3f-b3aa-f62b74103731","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721832454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45512024-01-24T19:07:24.105Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721856304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297,"session_id":"2f77e0af-7050-4d3f-b3aa-f62b74103731"}
45522024-01-24T19:07:24.105Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:22.721876361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1297}
45532024-01-24T19:07:24.105ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45542024-01-24T19:07:24.106ZLoading write count information from file "/tmp/test_repair_verify.data"
45552024-01-24T19:07:24.106ZRun Repair workload
45562024-01-24T19:07:24.106Z01/30 Write block 443 len 28672 data: 5 5 5 4 5 6 6
45572024-01-24T19:07:24.106Z02/30 Write block 427 len 12288 data: 4 4 4
45582024-01-24T19:07:24.106Z03/30 Read block 376 len 32768
45592024-01-24T19:07:24.106Z04/30 Write block 210 len 40960 data: 5 4 4 4 4 4 5 5 5 4
45602024-01-24T19:07:24.107Z05/30 Write block 557 len 12288 data: 4 4 4
45612024-01-24T19:07:24.107Z06/30 Read block 557 len 24576
45622024-01-24T19:07:24.107Z07/30 Write block 566 len 16384 data: 2 2 2 3
45632024-01-24T19:07:24.107Z08/30 Write block 50 len 36864 data: 5 4 4 4 4 5 6 5 6
45642024-01-24T19:07:24.107Z09/30 Flush
45652024-01-24T19:07:24.141Z10/30 Write block 76 len 28672 data: 4 5 5 6 6 8 6
45662024-01-24T19:07:24.141Z11/30 Read block 359 len 24576
45672024-01-24T19:07:24.142Z12/30 Read block 407 len 4096
45682024-01-24T19:07:24.142Z13/30 Write block 471 len 28672 data: 4 6 4 4 5 4 4
45692024-01-24T19:07:24.142Z14/30 Write block 243 len 36864 data: 5 5 6 8 6 5 4 3 2
45702024-01-24T19:07:24.142Z15/30 Read block 299 len 36864
45712024-01-24T19:07:24.142Z16/30 Read block 486 len 24576
45722024-01-24T19:07:24.142Z17/30 Read block 99 len 36864
45732024-01-24T19:07:24.142Z18/30 Read block 530 len 16384
45742024-01-24T19:07:24.142Z19/30 Write block 564 len 20480 data: 3 3 3 3 3
45752024-01-24T19:07:24.142Z20/30 Write block 294 len 12288 data: 6 4 3
45762024-01-24T19:07:24.142Z21/30 Write block 90 len 24576 data: 3 3 3 5 6 5
45772024-01-24T19:07:24.142Z22/30 Write block 185 len 28672 data: 6 6 5 5 4 3 3
45782024-01-24T19:07:24.143Z23/30 Read block 110 len 24576
45792024-01-24T19:07:24.143Z24/30 Read block 467 len 32768
45802024-01-24T19:07:24.143Z25/30 Read block 453 len 16384
45812024-01-24T19:07:24.143Z26/30 Read block 41 len 40960
45822024-01-24T19:07:24.143Z27/30 Write block 419 len 16384 data: 6 4 4 5
45832024-01-24T19:07:24.143Z28/30 Read block 372 len 40960
45842024-01-24T19:07:24.143Z29/30 Read block 495 len 20480
45852024-01-24T19:07:24.143Z30/30 Read block 380 len 40960
45862024-01-24T19:07:24.143Z----------------------------------------------------------------
45872024-01-24T19:07:24.143Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
45882024-01-24T19:07:24.143ZStates: Active Active Active
45892024-01-24T19:07:24.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
45902024-01-24T19:07:24.144Z 1 Acked 1000 Write 7 Done Done Sent false
45912024-01-24T19:07:24.144Z 2 Acked 1001 Write 3 Done Done Sent false
45922024-01-24T19:07:24.144Z 3 Acked 1002 Read 8 Done Done Sent false
45932024-01-24T19:07:24.144Z 4 Acked 1003 Write 10 Done Done Sent false
45942024-01-24T19:07:24.144Z 5 Acked 1004 Write 3 Done Done Sent false
45952024-01-24T19:07:24.144Z 6 Acked 1005 Read 6 Done Done Sent false
45962024-01-24T19:07:24.144Z 7 Acked 1006 Write 4 Done Done Sent false
45972024-01-24T19:07:24.144Z 8 Acked 1007 Write 9 Done Done Sent false
45982024-01-24T19:07:24.144Z 9 Acked 1008 Flush 0 Done Done Sent false
45992024-01-24T19:07:24.144Z 10 Acked 1009 Write 7 Done Done Sent false
46002024-01-24T19:07:24.144Z 11 Acked 1010 Read 6 Done Done Sent false
46012024-01-24T19:07:24.144Z 12 Acked 1011 Read 1 Done Done Sent false
46022024-01-24T19:07:24.145Z 13 Acked 1012 Write 7 Done Done Sent false
46032024-01-24T19:07:24.145Z 14 Acked 1013 Write 9 Done Done Sent false
46042024-01-24T19:07:24.145Z 15 Acked 1014 Read 9 Done Done Sent false
46052024-01-24T19:07:24.145Z 16 Acked 1015 Read 6 Done Done Sent false
46062024-01-24T19:07:24.145Z 17 Acked 1016 Read 9 Done Done Sent false
46072024-01-24T19:07:24.145Z 18 Acked 1017 Read 4 Done Done Sent false
46082024-01-24T19:07:24.145Z 19 Acked 1018 Write 5 Done Done Sent false
46092024-01-24T19:07:24.145Z 20 Acked 1019 Write 3 Done Done Sent false
46102024-01-24T19:07:24.145Z 21 Acked 1020 Write 6 Done Done Sent false
46112024-01-24T19:07:24.149Z 22 Acked 1021 Write 7 Done Done Sent false
46122024-01-24T19:07:24.150Z 23 Acked 1022 Read 6 Done Done Sent false
46132024-01-24T19:07:24.150Z 24 Acked 1023 Read 8 Done Done Sent false
46142024-01-24T19:07:24.150Z 25 Acked 1024 Read 4 Done Done Sent false
46152024-01-24T19:07:24.150Z 26 Acked 1025 Read 10 Done Done Sent false
46162024-01-24T19:07:24.150Z 27 Acked 1026 Write 4 Done Done Sent false
46172024-01-24T19:07:24.150Z 28 Acked 1027 Read 10 Done Done Sent false
46182024-01-24T19:07:24.150Z 29 Acked 1028 Read 5 Done Done Sent false
46192024-01-24T19:07:24.150Z 30 Acked 1029 Read 10 Done Sent Sent false
46202024-01-24T19:07:24.151Z STATES DS:0 DS:1 DS:2 TOTAL
46212024-01-24T19:07:24.151Z New 0 0 0 0
46222024-01-24T19:07:24.151Z Sent 0 1 30 31
46232024-01-24T19:07:24.151Z Done 30 29 0 59
46242024-01-24T19:07:24.151Z Skipped 0 0 0 0
46252024-01-24T19:07:24.151Z Error 0 0 0 0
46262024-01-24T19:07:24.152ZLast Flush: 1008 1008 0
46272024-01-24T19:07:24.152ZDownstairs last five completed:
46282024-01-24T19:07:24.152ZUpstairs last five completed: 30 29 28 27 26
46292024-01-24T19:07:24.152ZWrote out file "/tmp/test_repair_verify.data"
46302024-01-24T19:07:24.180Z
46312024-01-24T19:07:24.180ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46322024-01-24T19:07:24.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46332024-01-24T19:07:24.181Z 0 000-019 38 38 38 62 62 62 F F F
46342024-01-24T19:07:24.181Z 1 020-039 39 39 39 65 65 65 F F F
46352024-01-24T19:07:24.181Z 2 040-059 42 42 40 69 69 66 F F F <---
46362024-01-24T19:07:24.181Z 3 060-079 39 39 39 65 65 65 T T F <---
46372024-01-24T19:07:24.182Z 4 080-099 38 38 38 64 64 64 T T F <---
46382024-01-24T19:07:24.182Z 5 100-119 36 36 36 59 59 59 F F F
46392024-01-24T19:07:24.182Z 6 120-139 40 40 40 67 67 67 F F F
46402024-01-24T19:07:24.182Z 7 140-159 41 41 41 68 68 68 F F F
46412024-01-24T19:07:24.182Z 8 160-179 33 33 33 52 52 52 F F F
46422024-01-24T19:07:24.182Z 9 180-199 37 37 37 60 60 60 T T F <---
46432024-01-24T19:07:24.182Z 10 200-219 42 42 41 69 69 68 F F F <---
46442024-01-24T19:07:24.183Z 11 220-239 40 40 40 66 66 66 F F F
46452024-01-24T19:07:24.183Z 12 240-259 41 41 41 68 68 68 T T F <---
46462024-01-24T19:07:24.183Z 13 260-279 40 40 40 66 66 66 F F F
46472024-01-24T19:07:24.183Z 14 280-299 40 40 40 66 66 66 T T F <---
46482024-01-24T19:07:24.183Z 15 300-319 39 39 39 65 65 65 F F F
46492024-01-24T19:07:24.183Z 16 320-339 36 36 36 57 57 57 F F F
46502024-01-24T19:07:24.183Z 17 340-359 34 34 34 55 55 55 F F F
46512024-01-24T19:07:24.183Z 18 360-379 41 41 41 68 68 68 F F F
46522024-01-24T19:07:24.183Z 19 380-399 40 40 40 66 66 66 F F F
46532024-01-24T19:07:24.184Z 20 400-419 35 35 35 56 56 56 T T F <---
46542024-01-24T19:07:24.184Z 21 420-439 42 42 40 69 69 67 T T F <---
46552024-01-24T19:07:24.184Z 22 440-459 42 42 39 69 69 65 F F F <---
46562024-01-24T19:07:24.184Z 23 460-479 40 40 40 66 66 66 T T F <---
46572024-01-24T19:07:24.184Z 24 480-499 38 38 38 61 61 61 F F F
46582024-01-24T19:07:24.184Z 25 500-519 40 40 40 66 66 66 F F F
46592024-01-24T19:07:24.184Z 26 520-539 32 32 32 51 51 51 F F F
46602024-01-24T19:07:24.185Z 27 540-559 42 42 32 69 69 50 F F F <---
46612024-01-24T19:07:24.185Z 28 560-579 42 42 40 69 69 66 T T F <---
46622024-01-24T19:07:24.185Z 29 580-599 40 40 40 66 66 66 F F F
46632024-01-24T19:07:24.185ZMax gen: 42, Max flush: 69
46642024-01-24T19:07:24.185ZError: Difference in extent metadata found!
46652024-01-24T19:07:24.186ZA Difference in extent metadata is expected here
46662024-01-24T19:07:24.186ZOn loop 21
46672024-01-24T19:07:24.186Z
46682024-01-24T19:07:24.186ZVerifying data now
46692024-01-24T19:07:29.332ZNo change after verify
46702024-01-24T19:07:29.332ZLoop: 21 Downstairs dump after verify (and repair):
46712024-01-24T19:07:29.354ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46722024-01-24T19:07:29.354Z 0 000-019 38 38 38 62 62 62 F F F
46732024-01-24T19:07:29.355Z 1 020-039 39 39 39 65 65 65 F F F
46742024-01-24T19:07:29.355Z 2 040-059 42 42 42 69 69 69 F F F
46752024-01-24T19:07:29.355Z 3 060-079 43 43 43 70 70 70 F F F
46762024-01-24T19:07:29.355Z 4 080-099 43 43 43 70 70 70 F F F
46772024-01-24T19:07:29.355Z 5 100-119 36 36 36 59 59 59 F F F
46782024-01-24T19:07:29.355Z 6 120-139 40 40 40 67 67 67 F F F
46792024-01-24T19:07:29.356Z 7 140-159 41 41 41 68 68 68 F F F
46802024-01-24T19:07:29.356Z 8 160-179 33 33 33 52 52 52 F F F
46812024-01-24T19:07:29.356Z 9 180-199 43 43 43 70 70 70 F F F
46822024-01-24T19:07:29.370Z 10 200-219 42 42 42 69 69 69 F F F
46832024-01-24T19:07:29.370Z 11 220-239 40 40 40 66 66 66 F F F
46842024-01-24T19:07:29.370Z 12 240-259 43 43 43 70 70 70 F F F
46852024-01-24T19:07:29.371Z 13 260-279 40 40 40 66 66 66 F F F
46862024-01-24T19:07:29.371Z 14 280-299 43 43 43 70 70 70 F F F
46872024-01-24T19:07:29.371Z 15 300-319 39 39 39 65 65 65 F F F
46882024-01-24T19:07:29.371Z 16 320-339 36 36 36 57 57 57 F F F
46892024-01-24T19:07:29.371Z 17 340-359 34 34 34 55 55 55 F F F
46902024-01-24T19:07:29.371Z 18 360-379 41 41 41 68 68 68 F F F
46912024-01-24T19:07:29.371Z 19 380-399 40 40 40 66 66 66 F F F
46922024-01-24T19:07:29.371Z 20 400-419 43 43 43 70 70 70 F F F
46932024-01-24T19:07:29.372Z 21 420-439 43 43 43 70 70 70 F F F
46942024-01-24T19:07:29.372Z 22 440-459 42 42 42 69 69 69 F F F
46952024-01-24T19:07:29.372Z 23 460-479 43 43 43 70 70 70 F F F
46962024-01-24T19:07:29.372Z 24 480-499 38 38 38 61 61 61 F F F
46972024-01-24T19:07:29.372Z 25 500-519 40 40 40 66 66 66 F F F
46982024-01-24T19:07:29.372Z 26 520-539 32 32 32 51 51 51 F F F
46992024-01-24T19:07:29.372Z 27 540-559 42 42 42 69 69 69 F F F
47002024-01-24T19:07:29.373Z 28 560-579 43 43 43 70 70 70 F F F
47012024-01-24T19:07:29.373Z 29 580-599 40 40 40 66 66 66 F F F
47022024-01-24T19:07:29.373ZMax gen: 43, Max flush: 70
47032024-01-24T19:07:29.373Z
47042024-01-24T19:07:29.373ZBegin loop 22
47052024-01-24T19:07:29.373ZDownstairs to restart: 1
47062024-01-24T19:07:29.389Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.040558186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305}
47072024-01-24T19:07:29.389Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041010343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305}
47082024-01-24T19:07:29.389Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041056236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305}
47092024-01-24T19:07:29.411Z{"msg":"Crucible 2cc1b6a4-37bb-41a2-9a95-ecd003bc0b17 has session id: cc8cff0a-7554-4dee-be2e-51ae4f6c2daf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041106952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47102024-01-24T19:07:29.411Z{"msg":"Crucible runtime is spawned
47112024-01-24T19:07:29.411ZUpstairs opts: Upstairs UUID: 2cc1b6a4-37bb-41a2-9a95-ecd003bc0b17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041356718Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47122024-01-24T19:07:29.411Z{"msg":"Crucible stats registered with UUID: 2cc1b6a4-37bb-41a2-9a95-ecd003bc0b17","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.0414285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47132024-01-24T19:07:29.412Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041471602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305}
47142024-01-24T19:07:29.412Z{"msg":"2cc1b6a4-37bb-41a2-9a95-ecd003bc0b17 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041646446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47152024-01-24T19:07:29.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041712555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47162024-01-24T19:07:29.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041758316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47172024-01-24T19:07:29.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041799308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47182024-01-24T19:07:29.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041838305Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47192024-01-24T19:07:29.412Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041879597Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47202024-01-24T19:07:29.412Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.041919855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47212024-01-24T19:07:29.413Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042235036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47222024-01-24T19:07:29.413Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042377897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47232024-01-24T19:07:29.414Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042436008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47242024-01-24T19:07:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042475466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47252024-01-24T19:07:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042568579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47262024-01-24T19:07:29.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.042599551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47272024-01-24T19:07:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.04369165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47282024-01-24T19:07:29.414Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.043881515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47292024-01-24T19:07:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044018033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47302024-01-24T19:07:29.414Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044068289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47312024-01-24T19:07:29.414Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044125307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47322024-01-24T19:07:29.415Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044161153Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47332024-01-24T19:07:29.416Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044873858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47342024-01-24T19:07:29.416Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.044948033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47352024-01-24T19:07:29.416Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.045149807Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47362024-01-24T19:07:29.416Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.045930419Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47372024-01-24T19:07:29.416Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.045989647Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47382024-01-24T19:07:29.417Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046033497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47392024-01-24T19:07:29.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046223674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47402024-01-24T19:07:29.417Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046280701Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47412024-01-24T19:07:29.417Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046307411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47422024-01-24T19:07:29.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046349944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47432024-01-24T19:07:29.417Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046375554Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47442024-01-24T19:07:29.417Z{"msg":"[0]R flush_numbers[0..12]: [62, 65, 69, 70, 70, 59, 67, 68, 52, 70, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046419346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47452024-01-24T19:07:29.417Z{"msg":"[0]R generation[0..12]: [38, 39, 42, 43, 43, 36, 40, 41, 33, 43, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.04644599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47462024-01-24T19:07:29.417Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046470456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47472024-01-24T19:07:29.418Z{"msg":"[1]R flush_numbers[0..12]: [62, 65, 69, 70, 70, 59, 67, 68, 52, 70, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.04649326Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47482024-01-24T19:07:29.418Z{"msg":"[1]R generation[0..12]: [38, 39, 42, 43, 43, 36, 40, 41, 33, 43, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046540292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47492024-01-24T19:07:29.418Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046564157Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47502024-01-24T19:07:29.418Z{"msg":"[2]R flush_numbers[0..12]: [62, 65, 69, 70, 70, 59, 67, 68, 52, 70, 69, 66]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046631807Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47512024-01-24T19:07:29.418Z{"msg":"[2]R generation[0..12]: [38, 39, 42, 43, 43, 36, 40, 41, 33, 43, 42, 40]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
47522024-01-24T19:07:29.418Z,"time":"2024-01-24T19:07:28.046657697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47532024-01-24T19:07:29.419Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046681978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47542024-01-24T19:07:29.419Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046726335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47552024-01-24T19:07:29.420Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046751805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47562024-01-24T19:07:29.420Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046773047Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47572024-01-24T19:07:29.420Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046815134Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47582024-01-24T19:07:29.420Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046840639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47592024-01-24T19:07:29.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046881339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47602024-01-24T19:07:29.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046906367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47612024-01-24T19:07:29.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046950218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47622024-01-24T19:07:29.420Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.046975261Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47632024-01-24T19:07:29.420Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.047017517Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47642024-01-24T19:07:29.420Z{"msg":"2cc1b6a4-37bb-41a2-9a95-ecd003bc0b17 is now active with session: cc8cff0a-7554-4dee-be2e-51ae4f6c2daf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.047042478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47652024-01-24T19:07:29.420Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.047062667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305,"session_id":"cc8cff0a-7554-4dee-be2e-51ae4f6c2daf"}
47662024-01-24T19:07:29.421Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:28.047103481Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1305}
47672024-01-24T19:07:29.421ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47682024-01-24T19:07:29.421ZLoading write count information from file "/tmp/test_repair_verify.data"
47692024-01-24T19:07:29.421ZRun Repair workload
47702024-01-24T19:07:29.421Z01/30 Write block 571 len 24576 data: 3 3 2 3 4 4
47712024-01-24T19:07:29.421Z02/30 Flush
47722024-01-24T19:07:29.421Z03/30 Write block 327 len 20480 data: 4 4 4 3 3
47732024-01-24T19:07:29.422Z04/30 Read block 178 len 24576
47742024-01-24T19:07:29.422Z05/30 Write block 563 len 28672 data: 4 4 4 4 4 4 4
47752024-01-24T19:07:29.422Z06/30 Read block 576 len 8192
47762024-01-24T19:07:29.909Z07/30 Read block 101 len 28672
47772024-01-24T19:07:29.909Z08/30 Write block 523 len 32768 data: 4 5 6 6 5 5 4 4
47782024-01-24T19:07:29.909Z09/30 Read block 52 len 24576
47792024-01-24T19:07:29.909Z10/30 Read block 413 len 40960
47802024-01-24T19:07:29.909Z11/30 Flush
47812024-01-24T19:07:29.909Z12/30 Flush
47822024-01-24T19:07:29.910Z13/30 Write block 295 len 16384 data: 5 4 2 3
47832024-01-24T19:07:29.910Z14/30 Write block 510 len 36864 data: 4 5 5 7 6 6 4 6 7
47842024-01-24T19:07:29.910Z15/30 Write block 277 len 32768 data: 4 5 8 7 8 8 7 4
47852024-01-24T19:07:29.910Z16/30 Read block 400 len 36864
47862024-01-24T19:07:29.910Z17/30 Flush
47872024-01-24T19:07:29.910Z18/30 Write block 231 len 40960 data: 4 4 3 2 2 2 2 2 3 3
47882024-01-24T19:07:29.910Z19/30 Write block 175 len 40960 data: 5 5 3 3 3 5 5 5 4 5
47892024-01-24T19:07:29.911Z20/30 Write block 374 len 16384 data: 6 6 5 6
47902024-01-24T19:07:29.911Z21/30 Write block 26 len 8192 data: 6 7
47912024-01-24T19:07:29.911Z22/30 Read block 193 len 28672
47922024-01-24T19:07:29.911Z23/30 Read block 38 len 36864
47932024-01-24T19:07:29.947Z24/30 Read block 588 len 8192
47942024-01-24T19:07:29.947Z25/30 Flush
47952024-01-24T19:07:29.947Z26/30 Write block 221 len 40960 data: 4 6 3 5 7 7 7 6 4 4
47962024-01-24T19:07:29.947Z27/30 Read block 248 len 40960
47972024-01-24T19:07:29.947Z28/30 Write block 493 len 36864 data: 3 3 3 5 4 5 4 3 3
47982024-01-24T19:07:29.947Z29/30 Write block 339 len 8192 data: 6 6
47992024-01-24T19:07:29.948Z30/30 Read block 439 len 16384
48002024-01-24T19:07:29.948Z----------------------------------------------------------------
48012024-01-24T19:07:29.949Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:28
48022024-01-24T19:07:29.949ZStates: Active Active Active
48032024-01-24T19:07:29.949ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48042024-01-24T19:07:29.949Z 3 Acked 1002 Write 5 Done Sent Done false
48052024-01-24T19:07:29.949Z 4 Acked 1003 Read 6 Done Sent Done false
48062024-01-24T19:07:29.949Z 5 Acked 1004 Write 7 Done Sent Done false
48072024-01-24T19:07:29.949Z 6 Acked 1005 Read 2 Done Sent Done false
48082024-01-24T19:07:29.950Z 7 Acked 1006 Read 7 Done Sent Done false
48092024-01-24T19:07:29.950Z 8 Acked 1007 Write 8 Done Sent Done false
48102024-01-24T19:07:29.950Z 9 Acked 1008 Read 6 Done Sent Done false
48112024-01-24T19:07:29.950Z 10 Acked 1009 Read 10 Done Sent Done false
48122024-01-24T19:07:29.950Z 11 Acked 1010 Flush 0 Done Sent Done false
48132024-01-24T19:07:29.950Z 12 Acked 1011 Flush 0 Done Sent Done false
48142024-01-24T19:07:29.950Z 13 Acked 1012 Write 4 Done Sent Done false
48152024-01-24T19:07:29.950Z 14 Acked 1013 Write 9 Done Sent Done false
48162024-01-24T19:07:29.950Z 15 Acked 1014 Write 8 Done Sent Done false
48172024-01-24T19:07:29.951Z 16 Acked 1015 Read 9 Done Sent Done false
48182024-01-24T19:07:29.951Z 17 Acked 1016 Flush 0 Done Sent Done false
48192024-01-24T19:07:29.951Z 18 Acked 1017 Write 10 Done Sent Done false
48202024-01-24T19:07:29.951Z 19 Acked 1018 Write 10 Done Sent Done false
48212024-01-24T19:07:29.951Z 20 Acked 1019 Write 4 Done Sent Done false
48222024-01-24T19:07:29.951Z 21 Acked 1020 Write 2 Done Sent Done false
48232024-01-24T19:07:29.951Z 22 Acked 1021 Read 7 Done Sent Done false
48242024-01-24T19:07:29.966Z 23 Acked 1022 Read 9 Done Sent Done false
48252024-01-24T19:07:29.966Z 24 Acked 1023 Read 2 Done Sent Done false
48262024-01-24T19:07:29.966Z 25 Acked 1024 Flush 0 Done Sent Done false
48272024-01-24T19:07:29.967Z 26 Acked 1025 Write 10 Done Sent Done false
48282024-01-24T19:07:29.967Z 27 Acked 1026 Read 10 Done Sent Done false
48292024-01-24T19:07:29.967Z 28 Acked 1027 Write 9 Done Sent Done false
48302024-01-24T19:07:29.967Z 29 Acked 1028 Write 2 Done Sent Done false
48312024-01-24T19:07:29.967Z 30 Acked 1029 Read 4 Done Sent Done false
48322024-01-24T19:07:29.967Z STATES DS:0 DS:1 DS:2 TOTAL
48332024-01-24T19:07:29.968Z New 0 0 0 0
48342024-01-24T19:07:29.968Z Sent 0 28 0 28
48352024-01-24T19:07:29.990Z Done 28 0 28 56
48362024-01-24T19:07:29.990Z Skipped 0 0 0 0
48372024-01-24T19:07:29.990Z Error 0 0 0 0
48382024-01-24T19:07:29.991ZLast Flush: 1024 1001 1024
48392024-01-24T19:07:29.991ZDownstairs last five completed: 1001 1000
48402024-01-24T19:07:29.991ZUpstairs last five completed: 30 29 28 27 26
48412024-01-24T19:07:29.991ZWrote out file "/tmp/test_repair_verify.data"
48422024-01-24T19:07:29.991Z
48432024-01-24T19:07:29.991ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48442024-01-24T19:07:29.991ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48452024-01-24T19:07:29.992Z 0 000-019 38 38 38 62 62 62 F F F
48462024-01-24T19:07:29.992Z 1 020-039 44 39 44 75 65 75 F F F <---
48472024-01-24T19:07:29.992Z 2 040-059 42 42 42 69 69 69 F F F
48482024-01-24T19:07:29.992Z 3 060-079 43 43 43 70 70 70 F F F
48492024-01-24T19:07:29.992Z 4 080-099 43 43 43 70 70 70 F F F
48502024-01-24T19:07:29.992Z 5 100-119 36 36 36 59 59 59 F F F
48512024-01-24T19:07:29.993Z 6 120-139 40 40 40 67 67 67 F F F
48522024-01-24T19:07:29.993Z 7 140-159 41 41 41 68 68 68 F F F
48532024-01-24T19:07:29.993Z 8 160-179 44 33 44 75 52 75 F F F <---
48542024-01-24T19:07:29.993Z 9 180-199 44 43 44 75 70 75 F F F <---
48552024-01-24T19:07:29.993Z 10 200-219 42 42 42 69 69 69 F F F
48562024-01-24T19:07:29.994Z 11 220-239 44 40 44 75 66 75 T F T <---
48572024-01-24T19:07:29.994Z 12 240-259 44 43 44 75 70 75 F F F <---
48582024-01-24T19:07:29.994Z 13 260-279 44 40 44 74 66 74 F F F <---
48592024-01-24T19:07:29.994Z 14 280-299 44 43 44 74 70 74 F F F <---
48602024-01-24T19:07:29.994Z 15 300-319 39 39 39 65 65 65 F F F
48612024-01-24T19:07:29.994Z 16 320-339 44 36 44 72 57 72 T F T <---
48622024-01-24T19:07:29.994Z 17 340-359 34 34 34 55 55 55 T F T <---
48632024-01-24T19:07:29.995Z 18 360-379 44 41 44 75 68 75 F F F <---
48642024-01-24T19:07:29.995Z 19 380-399 40 40 40 66 66 66 F F F
48652024-01-24T19:07:29.995Z 20 400-419 43 43 43 70 70 70 F F F
48662024-01-24T19:07:29.995Z 21 420-439 43 43 43 70 70 70 F F F
48672024-01-24T19:07:29.995Z 22 440-459 42 42 42 69 69 69 F F F
48682024-01-24T19:07:29.995Z 23 460-479 43 43 43 70 70 70 F F F
48692024-01-24T19:07:29.995Z 24 480-499 38 38 38 61 61 61 T F T <---
48702024-01-24T19:07:29.996Z 25 500-519 44 40 44 74 66 74 T F T <---
48712024-01-24T19:07:29.996Z 26 520-539 44 32 44 72 51 72 F F F <---
48722024-01-24T19:07:29.996Z 27 540-559 42 42 42 69 69 69 F F F
48732024-01-24T19:07:29.997Z 28 560-579 44 44 44 72 71 72 F F F <---
48742024-01-24T19:07:29.997Z 29 580-599 40 40 40 66 66 66 F F F
48752024-01-24T19:07:29.997ZMax gen: 44, Max flush: 75
48762024-01-24T19:07:29.997ZError: Difference in extent metadata found!
48772024-01-24T19:07:29.997ZA Difference in extent metadata is expected here
48782024-01-24T19:07:29.998ZOn loop 22
48792024-01-24T19:07:29.998Z
48802024-01-24T19:07:29.998ZVerifying data now
48812024-01-24T19:07:35.022ZNo change after verify
48822024-01-24T19:07:35.022ZLoop: 22 Downstairs dump after verify (and repair):
48832024-01-24T19:07:35.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48842024-01-24T19:07:35.022Z 0 000-019 38 38 38 62 62 62 F F F
48852024-01-24T19:07:35.022Z 1 020-039 44 44 44 75 75 75 F F F
48862024-01-24T19:07:35.025Z 2 040-059 42 42 42 69 69 69 F F F
48872024-01-24T19:07:35.025Z 3 060-079 43 43 43 70 70 70 F F F
48882024-01-24T19:07:35.025Z 4 080-099 43 43 43 70 70 70 F F F
48892024-01-24T19:07:35.025Z 5 100-119 36 36 36 59 59 59 F F F
48902024-01-24T19:07:35.025Z 6 120-139 40 40 40 67 67 67 F F F
48912024-01-24T19:07:35.026Z 7 140-159 41 41 41 68 68 68 F F F
48922024-01-24T19:07:35.026Z 8 160-179 44 44 44 75 75 75 F F F
48932024-01-24T19:07:35.026Z 9 180-199 44 44 44 75 75 75 F F F
48942024-01-24T19:07:35.026Z 10 200-219 42 42 42 69 69 69 F F F
48952024-01-24T19:07:35.026Z 11 220-239 45 45 45 76 76 76 F F F
48962024-01-24T19:07:35.026Z 12 240-259 44 44 44 75 75 75 F F F
48972024-01-24T19:07:35.026Z 13 260-279 44 44 44 74 74 74 F F F
48982024-01-24T19:07:35.027Z 14 280-299 44 44 44 74 74 74 F F F
48992024-01-24T19:07:35.027Z 15 300-319 39 39 39 65 65 65 F F F
49002024-01-24T19:07:35.027Z 16 320-339 45 45 45 76 76 76 F F F
49012024-01-24T19:07:35.027Z 17 340-359 45 45 45 76 76 76 F F F
49022024-01-24T19:07:35.027Z 18 360-379 44 44 44 75 75 75 F F F
49032024-01-24T19:07:35.027Z 19 380-399 40 40 40 66 66 66 F F F
49042024-01-24T19:07:35.027Z 20 400-419 43 43 43 70 70 70 F F F
49052024-01-24T19:07:35.027Z 21 420-439 43 43 43 70 70 70 F F F
49062024-01-24T19:07:35.028Z 22 440-459 42 42 42 69 69 69 F F F
49072024-01-24T19:07:35.028Z 23 460-479 43 43 43 70 70 70 F F F
49082024-01-24T19:07:35.028Z 24 480-499 45 45 45 76 76 76 F F F
49092024-01-24T19:07:35.028Z 25 500-519 45 45 45 76 76 76 F F F
49102024-01-24T19:07:35.028Z 26 520-539 44 44 44 72 72 72 F F F
49112024-01-24T19:07:35.032Z 27 540-559 42 42 42 69 69 69 F F F
49122024-01-24T19:07:35.032Z 28 560-579 44 44 44 72 72 72 F F F
49132024-01-24T19:07:35.032Z 29 580-599 40 40 40 66 66 66 F F F
49142024-01-24T19:07:35.032ZMax gen: 45, Max flush: 76
49152024-01-24T19:07:35.032Z
49162024-01-24T19:07:35.033ZBegin loop 23
49172024-01-24T19:07:35.033ZDownstairs to restart: 2
49182024-01-24T19:07:35.033Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.429678404Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313}
49192024-01-24T19:07:35.033Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.4302169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313}
49202024-01-24T19:07:35.033Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430279729Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313}
49212024-01-24T19:07:35.033Z{"msg":"Crucible d45fe331-5137-44fe-9935-d9c873b04c22 has session id: f415f8b8-0bcf-46a5-a4a2-5bdfb778a729","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430343658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"Crucible runtime is spawned
49222024-01-24T19:07:35.033Zf415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49232024-01-24T19:07:35.033Z{"msg":"Upstairs opts: Upstairs UUID: d45fe331-5137-44fe-9935-d9c873b04c22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430508206Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49242024-01-24T19:07:35.033Z{"msg":"Crucible stats registered with UUID: d45fe331-5137-44fe-9935-d9c873b04c22","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430587697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49252024-01-24T19:07:35.033Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430637182Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313}
49262024-01-24T19:07:35.034Z{"msg":"d45fe331-5137-44fe-9935-d9c873b04c22 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430772332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49272024-01-24T19:07:35.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430831441Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49282024-01-24T19:07:35.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430887938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49292024-01-24T19:07:35.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430914555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49302024-01-24T19:07:35.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.43093739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49312024-01-24T19:07:35.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.430959317Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49322024-01-24T19:07:35.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431013236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49332024-01-24T19:07:35.034Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431054385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49342024-01-24T19:07:35.035Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431114075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49352024-01-24T19:07:35.035Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431194758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49362024-01-24T19:07:35.037Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431412046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49372024-01-24T19:07:35.037Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431540955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49382024-01-24T19:07:35.037Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.431614874Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49392024-01-24T19:07:35.037Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.43268574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49402024-01-24T19:07:35.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.432801429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49412024-01-24T19:07:35.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.4328732Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49422024-01-24T19:07:35.038Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.432922167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49432024-01-24T19:07:35.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.433088597Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49442024-01-24T19:07:35.038Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.433154316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49452024-01-24T19:07:35.038Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.433893043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49462024-01-24T19:07:35.038Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.433973912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49472024-01-24T19:07:35.038Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.434190887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49482024-01-24T19:07:35.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.434801114Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49492024-01-24T19:07:35.038Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.434917331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49502024-01-24T19:07:35.038Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.434995985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49512024-01-24T19:07:35.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435184353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49522024-01-24T19:07:35.076Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435251449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49532024-01-24T19:07:35.076Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435297979Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49542024-01-24T19:07:35.076Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435321966Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49552024-01-24T19:07:35.076Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435343893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49562024-01-24T19:07:35.076Z{"msg":"[0]R flush_numbers[0..12]: [62, 75, 69, 70, 70, 59, 67, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435387296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49572024-01-24T19:07:35.076Z{"msg":"[0]R generation[0..12]: [38, 44, 42, 43, 43, 36, 40, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.4354343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49582024-01-24T19:07:35.076Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435481972Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49592024-01-24T19:07:35.076Z{"msg":"[1]R flush_numbers[0..12]: [62, 75, 69, 70, 70, 59, 67, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435528487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49602024-01-24T19:07:35.076Z{"msg":"[1]R generation[0..12]: [38, 44, 42, 43, 43, 36, 40, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435576911Z","hostname":"Wait for a query_work_queue command to finish before sending IO
49612024-01-24T19:07:35.076Zip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49622024-01-24T19:07:35.076Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435609667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49632024-01-24T19:07:35.076Z{"msg":"[2]R flush_numbers[0..12]: [62, 75, 69, 70, 70, 59, 67, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435654255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49642024-01-24T19:07:35.077Z{"msg":"[2]R generation[0..12]: [38, 44, 42, 43, 43, 36, 40, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435701318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49652024-01-24T19:07:35.077Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435749018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49662024-01-24T19:07:35.078Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435794923Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49672024-01-24T19:07:35.078Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435841645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49682024-01-24T19:07:35.078Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435896978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49692024-01-24T19:07:35.078Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435943305Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49702024-01-24T19:07:35.078Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.435997542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49712024-01-24T19:07:35.078Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.43604179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49722024-01-24T19:07:35.078Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.436090109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49732024-01-24T19:07:35.078Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.436135451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49742024-01-24T19:07:35.078Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.436181877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49752024-01-24T19:07:35.078Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.436207164Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49762024-01-24T19:07:35.080Z{"msg":"d45fe331-5137-44fe-9935-d9c873b04c22 is now active with session: f415f8b8-0bcf-46a5-a4a2-5bdfb778a729","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
49772024-01-24T19:07:35.080Z,"time":"2024-01-24T19:07:33.436258792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49782024-01-24T19:07:35.080Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.43628314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313,"session_id":"f415f8b8-0bcf-46a5-a4a2-5bdfb778a729"}
49792024-01-24T19:07:35.080Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:33.436304519Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1313}
49802024-01-24T19:07:35.080ZLoading write count information from file "/tmp/test_repair_verify.data"
49812024-01-24T19:07:35.080ZRun Repair workload
49822024-01-24T19:07:35.080ZError: Difference in extent metadata found!
49832024-01-24T19:07:35.080Z01/30 Write block 70 len 16384 data: 3 3 4 4
49842024-01-24T19:07:35.080Z02/30 Read block 123 len 16384
49852024-01-24T19:07:35.080Z03/30 Write block 57 len 8192 data: 6 7
49862024-01-24T19:07:35.080Z04/30 Read block 556 len 12288
49872024-01-24T19:07:35.081Z05/30 Write block 433 len 8192 data: 4 4
49882024-01-24T19:07:35.081Z06/30 Read block 134 len 16384
49892024-01-24T19:07:35.081Z07/30 Read block 56 len 40960
49902024-01-24T19:07:35.081Z08/30 Read block 271 len 32768
49912024-01-24T19:07:35.081Z09/30 Read block 236 len 8192
49922024-01-24T19:07:35.081Z10/30 Flush
49932024-01-24T19:07:35.081Z11/30 Write block 332 len 36864 data: 4 4 4 4 4 6 6 7 7
49942024-01-24T19:07:35.081Z12/30 Read block 355 len 8192
49952024-01-24T19:07:35.081Z13/30 Write block 76 len 36864 data: 5 6 6 7 7 9 7 7 6
49962024-01-24T19:07:35.082Z14/30 Write block 546 len 12288 data: 3 4 5
49972024-01-24T19:07:35.082Z15/30 Read block 420 len 32768
49982024-01-24T19:07:35.082Z16/30 Read block 452 len 12288
49992024-01-24T19:07:35.082Z17/30 Read block 494 len 32768
50002024-01-24T19:07:35.082Z18/30 Read block 595 len 16384
50012024-01-24T19:07:35.082Z19/30 Read block 153 len 4096
50022024-01-24T19:07:35.082Z20/30 Write block 360 len 24576 data: 5 6 6 6 6 5
50032024-01-24T19:07:35.082Z21/30 Write block 100 len 36864 data: 5 3 3 5 5 5 6 7 8
50042024-01-24T19:07:35.082Z22/30 Write block 541 len 20480 data: 4 5 5 5 4
50052024-01-24T19:07:35.082Z23/30 Write block 380 len 16384 data: 5 4 3 4
50062024-01-24T19:07:35.082Z24/30 Write block 56 len 16384 data: 7 7 8 6
50072024-01-24T19:07:35.082Z25/30 Read block 13 len 32768
50082024-01-24T19:07:35.083Z26/30 Write block 287 len 20480 data: 4 5 5 5 5
50092024-01-24T19:07:35.083Z27/30 Read block 114 len 28672
50102024-01-24T19:07:35.083Z28/30 Read block 32 len 36864
50112024-01-24T19:07:35.083Z29/30 Read block 29 len 8192
50122024-01-24T19:07:35.083Z30/30 Write block 131 len 36864 data: 5 5 3 4 4 4 4 4 5
50132024-01-24T19:07:35.083Z----------------------------------------------------------------
50142024-01-24T19:07:35.083Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50152024-01-24T19:07:35.083ZStates: Active Active Active
50162024-01-24T19:07:35.083ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50172024-01-24T19:07:35.083Z 1 Acked 1000 Write 4 Done Done Done false
50182024-01-24T19:07:35.083Z 2 Acked 1001 Read 4 Done Done Done false
50192024-01-24T19:07:35.083Z 3 Acked 1002 Write 2 Done Done Done false
50202024-01-24T19:07:35.083Z 4 Acked 1003 Read 3 Done Done Done false
50212024-01-24T19:07:35.084Z 5 Acked 1004 Write 2 Done Done Sent false
50222024-01-24T19:07:35.084Z 6 Acked 1005 Read 4 Done Done Sent false
50232024-01-24T19:07:35.084Z 7 Acked 1006 Read 10 Done Done Sent false
50242024-01-24T19:07:35.084Z 8 Acked 1007 Read 8 Done Done Sent false
50252024-01-24T19:07:35.084Z 9 Acked 1008 Read 2 Done Done Sent false
50262024-01-24T19:07:35.084Z 10 Acked 1009 Flush 0 Done Done Sent false
50272024-01-24T19:07:35.084Z 11 Acked 1010 Write 9 Done Done Sent false
50282024-01-24T19:07:35.084Z 12 Acked 1011 Read 2 Done Done Sent false
50292024-01-24T19:07:35.084Z 13 Acked 1012 Write 9 Done Done Sent false
50302024-01-24T19:07:35.085Z 14 Acked 1013 Write 3 Done Done Sent false
50312024-01-24T19:07:35.085Z 15 Acked 1014 Read 8 Done Done Sent false
50322024-01-24T19:07:35.085Z 16 Acked 1015 Read 3 Done Done Sent false
50332024-01-24T19:07:35.085Z 17 Acked 1016 Read 8 Done Done Sent false
50342024-01-24T19:07:35.085Z 18 Acked 1017 Read 4 Done Done Sent false
50352024-01-24T19:07:35.085Z 19 Acked 1018 Read 1 Done Done Sent false
50362024-01-24T19:07:35.085Z 20 Acked 1019 Write 6 Done Done Sent false
50372024-01-24T19:07:35.086Z 21 Acked 1020 Write 9 Done Done Sent false
50382024-01-24T19:07:35.086Z 22 Acked 1021 Write 5 Done Done Sent false
50392024-01-24T19:07:35.086Z 23 Acked 1022 Write 4 Done Done Sent false
50402024-01-24T19:07:35.086Z 24 Acked 1023 Write 4 Done Done Sent false
50412024-01-24T19:07:35.086Z 25 Acked 1024 Read 8 Done Done Sent false
50422024-01-24T19:07:35.086Z 26 Acked 1025 Write 5 Done Done Sent false
50432024-01-24T19:07:35.086Z 27 Acked 1026 Read 7 Done Done Sent false
50442024-01-24T19:07:35.087Z 28 Acked 1027 Read 9 Done Done Sent false
50452024-01-24T19:07:35.087Z 29 Acked 1028 Read 2 Done Done Sent false
50462024-01-24T19:07:35.087Z 30 Acked 1029 Write 9 Sent Sent Sent false
50472024-01-24T19:07:35.087Z STATES DS:0 DS:1 DS:2 TOTAL
50482024-01-24T19:07:35.087Z New 0 0 0 0
50492024-01-24T19:07:35.087Z Sent 1 1 26 28
50502024-01-24T19:07:35.087Z Done 29 29 4 62
50512024-01-24T19:07:35.087Z Skipped 0 0 0 0
50522024-01-24T19:07:35.240Z Error 0 0 0 0
50532024-01-24T19:07:35.240ZLast Flush: 1009 1009 0
50542024-01-24T19:07:35.240ZDownstairs last five completed:
50552024-01-24T19:07:35.241ZUpstairs last five completed: 30 29 28 27 26
50562024-01-24T19:07:35.241ZWrote out file "/tmp/test_repair_verify.data"
50572024-01-24T19:07:35.241Z
50582024-01-24T19:07:35.241ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50592024-01-24T19:07:35.241ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50602024-01-24T19:07:35.242Z 0 000-019 38 38 38 62 62 62 F F F
50612024-01-24T19:07:35.242Z 1 020-039 44 44 44 75 75 75 F F F
50622024-01-24T19:07:35.242Z 2 040-059 46 46 42 77 77 69 T T T <---
50632024-01-24T19:07:35.242Z 3 060-079 46 46 43 77 77 70 T T T <---
50642024-01-24T19:07:35.242Z 4 080-099 43 43 43 70 70 70 T T F <---
50652024-01-24T19:07:35.243Z 5 100-119 36 36 36 59 59 59 T T F <---
50662024-01-24T19:07:35.243Z 6 120-139 40 40 40 67 67 67 T T F <---
50672024-01-24T19:07:35.243Z 7 140-159 41 41 41 68 68 68 F F F
50682024-01-24T19:07:35.244Z 8 160-179 44 44 44 75 75 75 F F F
50692024-01-24T19:07:35.244Z 9 180-199 44 44 44 75 75 75 F F F
50702024-01-24T19:07:35.244Z 10 200-219 42 42 42 69 69 69 F F F
50712024-01-24T19:07:35.244Z 11 220-239 45 45 45 76 76 76 F F F
50722024-01-24T19:07:35.244Z 12 240-259 44 44 44 75 75 75 F F F
50732024-01-24T19:07:35.245Z 13 260-279 44 44 44 74 74 74 F F F
50742024-01-24T19:07:35.245Z 14 280-299 44 44 44 74 74 74 T T F <---
50752024-01-24T19:07:35.245Z 15 300-319 39 39 39 65 65 65 F F F
50762024-01-24T19:07:35.245Z 16 320-339 45 45 45 76 76 76 T T F <---
50772024-01-24T19:07:35.245Z 17 340-359 45 45 45 76 76 76 T T F <---
50782024-01-24T19:07:35.245Z 18 360-379 44 44 44 75 75 75 T T F <---
50792024-01-24T19:07:35.246Z 19 380-399 40 40 40 66 66 66 T T F <---
50802024-01-24T19:07:35.246Z 20 400-419 43 43 43 70 70 70 F F F
50812024-01-24T19:07:35.246Z 21 420-439 46 46 43 77 77 70 F F F <---
50822024-01-24T19:07:35.246Z 22 440-459 42 42 42 69 69 69 F F F
50832024-01-24T19:07:35.246Z 23 460-479 43 43 43 70 70 70 F F F
50842024-01-24T19:07:35.246Z 24 480-499 45 45 45 76 76 76 F F F
50852024-01-24T19:07:35.246Z 25 500-519 45 45 45 76 76 76 F F F
50862024-01-24T19:07:35.246Z 26 520-539 44 44 44 72 72 72 F F F
50872024-01-24T19:07:35.247Z 27 540-559 42 42 42 69 69 69 T T F <---
50882024-01-24T19:07:35.247Z 28 560-579 44 44 44 72 72 72 F F F
50892024-01-24T19:07:35.247Z 29 580-599 40 40 40 66 66 66 F F F
50902024-01-24T19:07:35.248ZMax gen: 46, Max flush: 77
50912024-01-24T19:07:35.248ZA Difference in extent metadata is expected here
50922024-01-24T19:07:35.248ZOn loop 23
50932024-01-24T19:07:35.248Z
50942024-01-24T19:07:35.249ZVerifying data now
50952024-01-24T19:07:41.004ZNo change after verify
50962024-01-24T19:07:41.004ZLoop: 23 Downstairs dump after verify (and repair):
50972024-01-24T19:07:41.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50982024-01-24T19:07:41.422Z 0 000-019 38 38 38 62 62 62 F F F
50992024-01-24T19:07:41.446Z 1 020-039 44 44 44 75 75 75 F F F
51002024-01-24T19:07:41.446Z 2 040-059 47 47 47 78 78 78 F F F
51012024-01-24T19:07:41.447Z 3 060-079 47 47 47 78 78 78 F F F
51022024-01-24T19:07:41.447Z 4 080-099 47 47 47 78 78 78 F F F
51032024-01-24T19:07:41.447Z 5 100-119 47 47 47 78 78 78 F F F
51042024-01-24T19:07:41.447Z 6 120-139 47 47 47 78 78 78 F F F
51052024-01-24T19:07:41.447Z 7 140-159 41 41 41 68 68 68 F F F
51062024-01-24T19:07:41.448Z 8 160-179 44 44 44 75 75 75 F F F
51072024-01-24T19:07:41.448Z 9 180-199 44 44 44 75 75 75 F F F
51082024-01-24T19:07:41.448Z 10 200-219 42 42 42 69 69 69 F F F
51092024-01-24T19:07:41.448Z 11 220-239 45 45 45 76 76 76 F F F
51102024-01-24T19:07:41.448Z 12 240-259 44 44 44 75 75 75 F F F
51112024-01-24T19:07:41.448Z 13 260-279 44 44 44 74 74 74 F F F
51122024-01-24T19:07:41.448Z 14 280-299 47 47 47 78 78 78 F F F
51132024-01-24T19:07:41.449Z 15 300-319 39 39 39 65 65 65 F F F
51142024-01-24T19:07:41.449Z 16 320-339 47 47 47 78 78 78 F F F
51152024-01-24T19:07:41.449Z 17 340-359 47 47 47 78 78 78 F F F
51162024-01-24T19:07:41.449Z 18 360-379 47 47 47 78 78 78 F F F
51172024-01-24T19:07:41.449Z 19 380-399 47 47 47 78 78 78 F F F
51182024-01-24T19:07:41.449Z 20 400-419 43 43 43 70 70 70 F F F
51192024-01-24T19:07:41.449Z 21 420-439 46 46 46 77 77 77 F F F
51202024-01-24T19:07:41.450Z 22 440-459 42 42 42 69 69 69 F F F
51212024-01-24T19:07:41.450Z 23 460-479 43 43 43 70 70 70 F F F
51222024-01-24T19:07:41.451Z 24 480-499 45 45 45 76 76 76 F F F
51232024-01-24T19:07:41.451Z 25 500-519 45 45 45 76 76 76 F F F
51242024-01-24T19:07:41.451Z 26 520-539 44 44 44 72 72 72 F F F
51252024-01-24T19:07:41.451Z 27 540-559 47 47 47 78 78 78 F F F
51262024-01-24T19:07:41.451Z 28 560-579 44 44 44 72 72 72 F F F
51272024-01-24T19:07:41.451Z 29 580-599 40 40 40 66 66 66 F F F
51282024-01-24T19:07:41.451ZMax gen: 47, Max flush: 78
51292024-01-24T19:07:41.452Z
51302024-01-24T19:07:41.452ZBegin loop 24
51312024-01-24T19:07:41.452ZDownstairs to restart: 2
51322024-01-24T19:07:41.452Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.708398946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321}
51332024-01-24T19:07:41.452Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.70887411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321}
51342024-01-24T19:07:41.453Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.708904112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321}
51352024-01-24T19:07:41.453Z{"msg":"Crucible 671de77f-fd0f-4750-b22d-c5cc91999b00 has session id: 883081d5-b9c6-42e3-96b8-d4f624955462","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.708962312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51362024-01-24T19:07:41.453Z{"msg":"Crucible runtime is spawned
51372024-01-24T19:07:41.453ZUpstairs opts: Upstairs UUID: 671de77f-fd0f-4750-b22d-c5cc91999b00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.70911121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51382024-01-24T19:07:41.456Z{"msg":"Crucible stats registered with UUID: 671de77f-fd0f-4750-b22d-c5cc91999b00","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709243411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51392024-01-24T19:07:41.456Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709275399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321}
51402024-01-24T19:07:41.457Z{"msg":"671de77f-fd0f-4750-b22d-c5cc91999b00 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709508815Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51412024-01-24T19:07:41.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709565753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51422024-01-24T19:07:41.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709593826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51432024-01-24T19:07:41.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709616032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51442024-01-24T19:07:41.457Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709695674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51452024-01-24T19:07:41.457Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.709905565Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51462024-01-24T19:07:41.458Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.710095507Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51472024-01-24T19:07:41.458Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.710173555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51482024-01-24T19:07:41.458Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.710219423Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51492024-01-24T19:07:41.458Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71026375Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51502024-01-24T19:07:41.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71033768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51512024-01-24T19:07:41.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.710368252Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51522024-01-24T19:07:41.458Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71044004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51532024-01-24T19:07:41.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.711414676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51542024-01-24T19:07:41.459Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.711483112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51552024-01-24T19:07:41.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.711595021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51562024-01-24T19:07:41.459Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71187171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51572024-01-24T19:07:41.459Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71207679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51582024-01-24T19:07:41.459Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.712134867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51592024-01-24T19:07:41.459Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.712887808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51602024-01-24T19:07:41.459Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.7129643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51612024-01-24T19:07:41.460Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.713065373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51622024-01-24T19:07:41.460Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.713607935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51632024-01-24T19:07:41.460Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.713677524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51642024-01-24T19:07:41.460Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.713726926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51652024-01-24T19:07:41.460Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.713920604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51662024-01-24T19:07:41.460Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714023308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51672024-01-24T19:07:41.461Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714078079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51682024-01-24T19:07:41.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714117896Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51692024-01-24T19:07:41.462Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714173831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51702024-01-24T19:07:41.462Z{"msg":"[0]R flush_numbers[0..12]: [62, 75, 78, 78, 78, 78, 78, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71421927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51712024-01-24T19:07:41.462Z{"msg":"[0]R generation[0..12]: [38, 44, 47, 47, 47, 47, 47, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714277056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51722024-01-24T19:07:41.463Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714323338Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51732024-01-24T19:07:41.463Z{"msg":"[1]R flush_numbers[0..12]: [62, 75, 78, 78, 78, 78, 78, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
51742024-01-24T19:07:41.463Ztime":"2024-01-24T19:07:39.714382449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51752024-01-24T19:07:41.463Z{"msg":"[1]R generation[0..12]: [38, 44, 47, 47, 47, 47, 47, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714431483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51762024-01-24T19:07:41.463Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71447651Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51772024-01-24T19:07:41.463Z{"msg":"[2]R flush_numbers[0..12]: [62, 75, 78, 78, 78, 78, 78, 68, 75, 75, 69, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714535452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51782024-01-24T19:07:41.464Z{"msg":"[2]R generation[0..12]: [38, 44, 47, 47, 47, 47, 47, 41, 44, 44, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714618821Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51792024-01-24T19:07:41.464Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714663031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51802024-01-24T19:07:41.464Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714722062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51812024-01-24T19:07:41.464Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714766436Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51822024-01-24T19:07:41.464Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714821262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51832024-01-24T19:07:41.464Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714863669Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51842024-01-24T19:07:41.465Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714919896Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51852024-01-24T19:07:41.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.714959218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51862024-01-24T19:07:41.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715027359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51872024-01-24T19:07:41.465Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715071703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51882024-01-24T19:07:41.466Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715127854Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51892024-01-24T19:07:41.466Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715167787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51902024-01-24T19:07:41.467ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
51912024-01-24T19:07:41.467Z{"msg":"671de77f-fd0f-4750-b22d-c5cc91999b00 is now active with session: 883081d5-b9c6-42e3-96b8-d4f624955462","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.71535226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51922024-01-24T19:07:41.467Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715444588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321,"session_id":"883081d5-b9c6-42e3-96b8-d4f624955462"}
51932024-01-24T19:07:41.467Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:39.715519Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1321}
51942024-01-24T19:07:41.467ZLoading write count information from file "/tmp/test_repair_verify.data"
51952024-01-24T19:07:41.467ZRun Repair workload
51962024-01-24T19:07:41.467Z01/30 Write block 231 len 16384 data: 5 5 4 3
51972024-01-24T19:07:41.467Z02/30 Read block 98 len 12288
51982024-01-24T19:07:41.467Z03/30 Read block 0 len 20480
51992024-01-24T19:07:41.488Z04/30 Read block 1 len 36864
52002024-01-24T19:07:41.488Z05/30 Write block 249 len 36864 data: 5 4 3 2 2 3 3 3 3
52012024-01-24T19:07:41.488Z06/30 Read block 380 len 8192
52022024-01-24T19:07:41.488Z07/30 Write block 261 len 8192 data: 3 2
52032024-01-24T19:07:41.488Z08/30 Read block 200 len 40960
52042024-01-24T19:07:41.488Z09/30 Flush
52052024-01-24T19:07:41.488Z10/30 Flush
52062024-01-24T19:07:41.491Z11/30 Flush
52072024-01-24T19:07:41.491Z12/30 Flush
52082024-01-24T19:07:41.491Z13/30 Write block 19 len 16384 data: 2 3 3 3
52092024-01-24T19:07:41.491Z14/30 Read block 402 len 8192
52102024-01-24T19:07:41.492Z15/30 Read block 514 len 40960
52112024-01-24T19:07:41.492Z16/30 Write block 586 len 36864 data: 4 4 4 4 3 5 7 6 5
52122024-01-24T19:07:41.492Z17/30 Read block 250 len 20480
52132024-01-24T19:07:41.492Z18/30 Read block 275 len 28672
52142024-01-24T19:07:41.492Z19/30 Read block 128 len 8192
52152024-01-24T19:07:41.492Z20/30 Write block 273 len 32768 data: 4 3 2 2 5 6 9 8
52162024-01-24T19:07:41.492Z21/30 Write block 428 len 20480 data: 5 5 4 4 4
52172024-01-24T19:07:41.492Z22/30 Read block 433 len 36864
52182024-01-24T19:07:41.492Z23/30 Read block 528 len 16384
52192024-01-24T19:07:41.492Z24/30 Read block 531 len 36864
52202024-01-24T19:07:41.492Z25/30 Write block 422 len 8192 data: 6 5
52212024-01-24T19:07:41.492Z26/30 Read block 148 len 4096
52222024-01-24T19:07:41.493Z27/30 Read block 438 len 20480
52232024-01-24T19:07:41.493Z28/30 Read block 113 len 12288
52242024-01-24T19:07:41.493Z29/30 Read block 58 len 12288
52252024-01-24T19:07:41.493Z30/30 Write block 249 len 32768 data: 6 5 4 3 3 4 4 4
52262024-01-24T19:07:41.493Z----------------------------------------------------------------
52272024-01-24T19:07:41.493Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52282024-01-24T19:07:41.493ZStates: Active Active Active
52292024-01-24T19:07:41.493ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52302024-01-24T19:07:41.493Z 1 Acked 1000 Write 4 Done Done Done false
52312024-01-24T19:07:41.493Z 2 Acked 1001 Read 3 Done Done Done false
52322024-01-24T19:07:41.495Z 3 Acked 1002 Read 5 Done Done Done false
52332024-01-24T19:07:41.495Z 4 Acked 1003 Read 9 Done Done Done false
52342024-01-24T19:07:41.495Z 5 Acked 1004 Write 9 Done Done Done false
52352024-01-24T19:07:41.495Z 6 Acked 1005 Read 2 Done Done Done false
52362024-01-24T19:07:41.495Z 7 Acked 1006 Write 2 Done Done Done false
52372024-01-24T19:07:41.495Z 8 Acked 1007 Read 10 Done Done Done false
52382024-01-24T19:07:41.495Z 9 Acked 1008 Flush 0 Done Done Sent false
52392024-01-24T19:07:41.496Z 10 Acked 1009 Flush 0 Done Done Sent false
52402024-01-24T19:07:41.496Z 11 Acked 1010 Flush 0 Done Done Sent false
52412024-01-24T19:07:41.496Z 12 Acked 1011 Flush 0 Done Done Sent false
52422024-01-24T19:07:41.497Z 13 Acked 1012 Write 4 Done Done Sent false
52432024-01-24T19:07:41.497Z 14 Acked 1013 Read 2 Done Done Sent false
52442024-01-24T19:07:41.497Z 15 Acked 1014 Read 10 Done Done Sent false
52452024-01-24T19:07:41.497Z 16 Acked 1015 Write 9 Done Done Sent false
52462024-01-24T19:07:41.497Z 17 Acked 1016 Read 5 Done Done Sent false
52472024-01-24T19:07:41.497Z 18 Acked 1017 Read 7 Done Done Sent false
52482024-01-24T19:07:41.497Z 19 Acked 1018 Read 2 Done Done Sent false
52492024-01-24T19:07:41.497Z 20 Acked 1019 Write 8 Done Done Sent false
52502024-01-24T19:07:41.497Z 21 Acked 1020 Write 5 Done Done Sent false
52512024-01-24T19:07:41.497Z 22 Acked 1021 Read 9 Done Done Sent false
52522024-01-24T19:07:41.497Z 23 Acked 1022 Read 4 Done Done Sent false
52532024-01-24T19:07:41.497Z 24 Acked 1023 Read 9 Done Done Sent false
52542024-01-24T19:07:41.498Z 25 Acked 1024 Write 2 Done Done Sent false
52552024-01-24T19:07:41.498Z 26 Acked 1025 Read 1 Done Done Sent false
52562024-01-24T19:07:41.498Z 27 Acked 1026 Read 5 Done Done Sent false
52572024-01-24T19:07:41.498Z 28 Acked 1027 Read 3 Done Done Sent false
52582024-01-24T19:07:41.498Z 29 Acked 1028 Read 3 Done Done Sent false
52592024-01-24T19:07:41.498Z 30 Acked 1029 Write 8 Sent Sent Sent false
52602024-01-24T19:07:41.498Z STATES DS:0 DS:1 DS:2 TOTAL
52612024-01-24T19:07:41.498Z New 0 0 0 0
52622024-01-24T19:07:41.498Z Sent 1 1 22 24
52632024-01-24T19:07:41.498Z Done 29 29 8 66
52642024-01-24T19:07:41.498Z Skipped 0 0 0 0
52652024-01-24T19:07:41.499Z Error 0 0 0 0
52662024-01-24T19:07:41.499ZLast Flush: 1011 1011 0
52672024-01-24T19:07:41.499ZDownstairs last five completed:
52682024-01-24T19:07:41.499ZUpstairs last five completed: 30 29 28 27 26
52692024-01-24T19:07:41.499ZWrote out file "/tmp/test_repair_verify.data"
52702024-01-24T19:07:41.499Z
52712024-01-24T19:07:41.499ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
52722024-01-24T19:07:41.499ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52732024-01-24T19:07:41.499Z 0 000-019 38 38 38 62 62 62 T T F <---
52742024-01-24T19:07:41.499Z 1 020-039 44 44 44 75 75 75 T T F <---
52752024-01-24T19:07:41.499Z 2 040-059 47 47 47 78 78 78 F F F
52762024-01-24T19:07:41.500Z 3 060-079 47 47 47 78 78 78 F F F
52772024-01-24T19:07:41.500Z 4 080-099 47 47 47 78 78 78 F F F
52782024-01-24T19:07:41.500Z 5 100-119 47 47 47 78 78 78 F F F
52792024-01-24T19:07:41.500Z 6 120-139 47 47 47 78 78 78 F F F
52802024-01-24T19:07:41.500Z 7 140-159 41 41 41 68 68 68 F F F
52812024-01-24T19:07:41.500Z 8 160-179 44 44 44 75 75 75 F F F
52822024-01-24T19:07:41.500Z 9 180-199 44 44 44 75 75 75 F F F
52832024-01-24T19:07:41.500Z 10 200-219 42 42 42 69 69 69 F F F
52842024-01-24T19:07:41.500Z 11 220-239 48 48 45 79 79 76 F F T <---
52852024-01-24T19:07:41.500Z 12 240-259 48 48 44 79 79 75 T T T <---
52862024-01-24T19:07:41.500Z 13 260-279 48 48 44 79 79 74 T T T <---
52872024-01-24T19:07:41.500Z 14 280-299 47 47 47 78 78 78 T T F <---
52882024-01-24T19:07:41.500Z 15 300-319 39 39 39 65 65 65 F F F
52892024-01-24T19:07:41.500Z 16 320-339 47 47 47 78 78 78 F F F
52902024-01-24T19:07:41.501Z 17 340-359 47 47 47 78 78 78 F F F
52912024-01-24T19:07:41.501Z 18 360-379 47 47 47 78 78 78 F F F
52922024-01-24T19:07:41.501Z 19 380-399 47 47 47 78 78 78 F F F
52932024-01-24T19:07:41.501Z 20 400-419 43 43 43 70 70 70 F F F
52942024-01-24T19:07:41.501Z 21 420-439 46 46 46 77 77 77 T T F <---
52952024-01-24T19:07:41.501Z 22 440-459 42 42 42 69 69 69 F F F
52962024-01-24T19:07:41.501Z 23 460-479 43 43 43 70 70 70 F F F
52972024-01-24T19:07:41.501Z 24 480-499 45 45 45 76 76 76 F F F
52982024-01-24T19:07:41.501Z 25 500-519 45 45 45 76 76 76 F F F
52992024-01-24T19:07:41.517Z 26 520-539 44 44 44 72 72 72 F F F
53002024-01-24T19:07:41.517Z 27 540-559 47 47 47 78 78 78 F F F
53012024-01-24T19:07:41.517Z 28 560-579 44 44 44 72 72 72 F F F
53022024-01-24T19:07:41.517Z 29 580-599 40 40 40 66 66 66 T T F <---
53032024-01-24T19:07:41.517ZMax gen: 48, Max flush: 79
53042024-01-24T19:07:41.517ZError: Difference in extent metadata found!
53052024-01-24T19:07:41.519ZA Difference in extent metadata is expected here
53062024-01-24T19:07:41.519ZOn loop 24
53072024-01-24T19:07:41.519Z
53082024-01-24T19:07:41.519ZVerifying data now
53092024-01-24T19:07:46.940ZNo change after verify
53102024-01-24T19:07:46.940ZLoop: 24 Downstairs dump after verify (and repair):
53112024-01-24T19:07:46.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53122024-01-24T19:07:46.969Z 0 000-019 49 49 49 80 80 80 F F F
53132024-01-24T19:07:46.977Z 1 020-039 49 49 49 80 80 80 F F F
53142024-01-24T19:07:46.977Z 2 040-059 47 47 47 78 78 78 F F F
53152024-01-24T19:07:46.977Z 3 060-079 47 47 47 78 78 78 F F F
53162024-01-24T19:07:46.978Z 4 080-099 47 47 47 78 78 78 F F F
53172024-01-24T19:07:46.978Z 5 100-119 47 47 47 78 78 78 F F F
53182024-01-24T19:07:46.978Z 6 120-139 47 47 47 78 78 78 F F F
53192024-01-24T19:07:46.978Z 7 140-159 41 41 41 68 68 68 F F F
53202024-01-24T19:07:46.978Z 8 160-179 44 44 44 75 75 75 F F F
53212024-01-24T19:07:46.978Z 9 180-199 44 44 44 75 75 75 F F F
53222024-01-24T19:07:46.979Z 10 200-219 42 42 42 69 69 69 F F F
53232024-01-24T19:07:46.979Z 11 220-239 48 48 48 79 79 79 F F F
53242024-01-24T19:07:46.979Z 12 240-259 49 49 49 80 80 80 F F F
53252024-01-24T19:07:46.979Z 13 260-279 49 49 49 80 80 80 F F F
53262024-01-24T19:07:46.979Z 14 280-299 49 49 49 80 80 80 F F F
53272024-01-24T19:07:46.981Z 15 300-319 39 39 39 65 65 65 F F F
53282024-01-24T19:07:46.981Z 16 320-339 47 47 47 78 78 78 F F F
53292024-01-24T19:07:46.981Z 17 340-359 47 47 47 78 78 78 F F F
53302024-01-24T19:07:46.981Z 18 360-379 47 47 47 78 78 78 F F F
53312024-01-24T19:07:46.981Z 19 380-399 47 47 47 78 78 78 F F F
53322024-01-24T19:07:46.981Z 20 400-419 43 43 43 70 70 70 F F F
53332024-01-24T19:07:46.981Z 21 420-439 49 49 49 80 80 80 F F F
53342024-01-24T19:07:46.982Z 22 440-459 42 42 42 69 69 69 F F F
53352024-01-24T19:07:46.982Z 23 460-479 43 43 43 70 70 70 F F F
53362024-01-24T19:07:46.982Z 24 480-499 45 45 45 76 76 76 F F F
53372024-01-24T19:07:46.982Z 25 500-519 45 45 45 76 76 76 F F F
53382024-01-24T19:07:46.982Z 26 520-539 44 44 44 72 72 72 F F F
53392024-01-24T19:07:46.982Z 27 540-559 47 47 47 78 78 78 F F F
53402024-01-24T19:07:46.984Z 28 560-579 44 44 44 72 72 72 F F F
53412024-01-24T19:07:46.984Z 29 580-599 49 49 49 80 80 80 F F F
53422024-01-24T19:07:46.984ZMax gen: 49, Max flush: 80
53432024-01-24T19:07:46.984Z
53442024-01-24T19:07:46.984ZBegin loop 25
53452024-01-24T19:07:46.984ZDownstairs to restart: 1
53462024-01-24T19:07:47.000Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.656821152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329}
53472024-01-24T19:07:47.000Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.65727096Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329}
53482024-01-24T19:07:47.024Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.657346478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329}
53492024-01-24T19:07:47.024Z{"msg":"Crucible 7762dec1-4554-4809-a879-6e565dfd7494 has session id: 2a79f62d-248e-4375-b9c5-55c691bcab78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.657452079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53502024-01-24T19:07:47.024Z{"msg":"Upstairs opts: Upstairs UUID: 7762dec1-4554-4809-a879-6e565dfd7494, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.657514495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53512024-01-24T19:07:47.024Z{"msg":"Crucible stats registered with UUID: 7762dec1-4554-4809-a879-6e565dfd7494","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.65759785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53522024-01-24T19:07:47.024ZCrucible runtime is spawned
53532024-01-24T19:07:47.024Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.657838336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329}
53542024-01-24T19:07:47.025Z{"msg":"7762dec1-4554-4809-a879-6e565dfd7494 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.657999621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53552024-01-24T19:07:47.025Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658223822Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53562024-01-24T19:07:47.025Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658324656Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53572024-01-24T19:07:47.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658365759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53582024-01-24T19:07:47.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658437654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53592024-01-24T19:07:47.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658480146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53602024-01-24T19:07:47.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658520545Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53612024-01-24T19:07:47.027Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658594127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53622024-01-24T19:07:47.027Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658642438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53632024-01-24T19:07:47.027Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658737555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53642024-01-24T19:07:47.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658783009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53652024-01-24T19:07:47.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658823311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53662024-01-24T19:07:47.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.658949816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53672024-01-24T19:07:47.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.660109185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53682024-01-24T19:07:47.028Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.660198734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53692024-01-24T19:07:47.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.660277424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53702024-01-24T19:07:47.028Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66032759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53712024-01-24T19:07:47.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66040099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53722024-01-24T19:07:47.029Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.660534603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53732024-01-24T19:07:47.029Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.661245053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53742024-01-24T19:07:47.029Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.661328099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53752024-01-24T19:07:47.029Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66137739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53762024-01-24T19:07:47.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66211267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53772024-01-24T19:07:47.030Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662174086Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53782024-01-24T19:07:47.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662201506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53792024-01-24T19:07:47.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662369985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53802024-01-24T19:07:47.031Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66246066Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53812024-01-24T19:07:47.031Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662486977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53822024-01-24T19:07:47.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66253823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53832024-01-24T19:07:47.031Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662564926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53842024-01-24T19:07:47.031Z{"msg":"[0]R flush_numbers[0..12]: [80, 80, 78, 78, 78, 78, 78, 68, 75, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662609009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53852024-01-24T19:07:47.032Z{"msg":"[0]R generation[0..12]: [49, 49, 47, 47, 47, 47, 47, 41, 44, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662635209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53862024-01-24T19:07:47.032Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662679796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53872024-01-24T19:07:47.033Z{"msg":"[1]R flush_numbers[0..12]: [80, 80, 78, 78, 78, 78, 78, 68, 75, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662705751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53882024-01-24T19:07:47.033Z{"msg":"[1]R generation[0..12]: [49, 49, 47, 47, 47, 47, 47, 41, 44, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
53892024-01-24T19:07:47.033Z2024-01-24T19:07:45.662749287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53902024-01-24T19:07:47.033Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662780271Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53912024-01-24T19:07:47.034Z{"msg":"[2]R flush_numbers[0..12]: [80, 80, 78, 78, 78, 78, 78, 68, 75, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662802776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53922024-01-24T19:07:47.034Z{"msg":"[2]R generation[0..12]: [49, 49, 47, 47, 47, 47, 47, 41, 44, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.66283085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53932024-01-24T19:07:47.034Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662897705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53942024-01-24T19:07:47.034Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662924474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53952024-01-24T19:07:47.035Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.662959147Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53962024-01-24T19:07:47.035Z{"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663021923Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53972024-01-24T19:07:47.036Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663049441Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53982024-01-24T19:07:47.036Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663071188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
53992024-01-24T19:07:47.036Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663112691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54002024-01-24T19:07:47.036Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663137886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54012024-01-24T19:07:47.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663186319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54022024-01-24T19:07:47.037Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663214036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54032024-01-24T19:07:47.037Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663261134Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54042024-01-24T19:07:47.037Z{"msg":"7762dec1-4554-4809-a879-6e565dfd7494 is now active with session: 2a79f62d-248e-4375-b9c5-55c691bcab78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663286464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54052024-01-24T19:07:47.037Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663340919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329,"session_id":"2a79f62d-248e-4375-b9c5-55c691bcab78"}
54062024-01-24T19:07:47.037Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:45.663366734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1329}
54072024-01-24T19:07:47.038ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54082024-01-24T19:07:47.038ZLoading write count information from file "/tmp/test_repair_verify.data"
54092024-01-24T19:07:47.038ZRun Repair workload
54102024-01-24T19:07:47.038Z01/30 Write block 33 len 32768 data: 7 6 2 2 2 2 2 3
54112024-01-24T19:07:47.087Z02/30 Write block 427 len 28672 data: 5 6 6 5 5 5 5
54122024-01-24T19:07:47.087Z03/30 Write block 472 len 12288 data: 7 5 5
54132024-01-24T19:07:47.088Z04/30 Read block 512 len 4096
54142024-01-24T19:07:47.088Z05/30 Write block 548 len 16384 data: 6 5 4 4
54152024-01-24T19:07:47.088Z06/30 Write block 161 len 4096 data: 4
54162024-01-24T19:07:47.088Z07/30 Flush
54172024-01-24T19:07:47.089Z08/30 Write block 161 len 36864 data: 5 5 5 4 4 3 4 4 4
54182024-01-24T19:07:47.089Z09/30 Read block 332 len 28672
54192024-01-24T19:07:47.089Z10/30 Read block 488 len 36864
54202024-01-24T19:07:47.089Z11/30 Flush
54212024-01-24T19:07:47.089Z12/30 Write block 542 len 24576 data: 6 6 6 5 4 5
54222024-01-24T19:07:47.089Z13/30 Read block 188 len 32768
54232024-01-24T19:07:47.090Z14/30 Read block 331 len 4096
54242024-01-24T19:07:47.090Z15/30 Write block 288 len 28672 data: 6 6 6 6 5 6 7
54252024-01-24T19:07:47.090Z16/30 Read block 402 len 4096
54262024-01-24T19:07:47.090Z17/30 Read block 526 len 16384
54272024-01-24T19:07:47.091Z18/30 Read block 295 len 40960
54282024-01-24T19:07:47.091Z19/30 Write block 241 len 32768 data: 4 4 6 6 7 9 7 6
54292024-01-24T19:07:47.091Z20/30 Read block 18 len 4096
54302024-01-24T19:07:47.091Z21/30 Read block 34 len 16384
54312024-01-24T19:07:47.091Z22/30 Write block 138 len 12288 data: 5 6 5
54322024-01-24T19:07:47.092Z23/30 Read block 466 len 20480
54332024-01-24T19:07:47.092Z24/30 Flush
54342024-01-24T19:07:47.092Z25/30 Flush
54352024-01-24T19:07:47.092Z26/30 Write block 332 len 28672 data: 5 5 5 5 5 7 7
54362024-01-24T19:07:47.092Z27/30 Write block 405 len 16384 data: 4 4 4 4
54372024-01-24T19:07:47.093Z28/30 Read block 189 len 20480
54382024-01-24T19:07:47.093Z29/30 Write block 29 len 24576 data: 7 7 8 7 8 7
54392024-01-24T19:07:47.093Z30/30 Read block 112 len 4096
54402024-01-24T19:07:47.093Z----------------------------------------------------------------
54412024-01-24T19:07:47.099Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54422024-01-24T19:07:47.099ZStates: Active Active Active
54432024-01-24T19:07:47.100ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54442024-01-24T19:07:47.100Z 1 Acked 1000 Write 8 Done Done Done false
54452024-01-24T19:07:47.100Z 2 Acked 1001 Write 7 Done Done Done false
54462024-01-24T19:07:47.100Z 3 Acked 1002 Write 3 Done Done Done false
54472024-01-24T19:07:47.100Z 4 Acked 1003 Read 1 Done Done Done false
54482024-01-24T19:07:47.100Z 5 Acked 1004 Write 4 Done Done Done false
54492024-01-24T19:07:47.101Z 6 Acked 1005 Write 1 Done Sent Done false
54502024-01-24T19:07:47.101Z 7 Acked 1006 Flush 0 Done Sent Done false
54512024-01-24T19:07:47.101Z 8 Acked 1007 Write 9 Done Sent Done false
54522024-01-24T19:07:47.101Z 9 Acked 1008 Read 7 Done Sent Done false
54532024-01-24T19:07:47.101Z 10 Acked 1009 Read 9 Done Sent Done false
54542024-01-24T19:07:47.101Z 11 Acked 1010 Flush 0 Done Sent Done false
54552024-01-24T19:07:47.101Z 12 Acked 1011 Write 6 Done Sent Done false
54562024-01-24T19:07:47.101Z 13 Acked 1012 Read 8 Done Sent Done false
54572024-01-24T19:07:47.102Z 14 Acked 1013 Read 1 Done Sent Done false
54582024-01-24T19:07:47.108Z 15 Acked 1014 Write 7 Done Sent Done false
54592024-01-24T19:07:47.108Z 16 Acked 1015 Read 1 Done Sent Done false
54602024-01-24T19:07:47.108Z 17 Acked 1016 Read 4 Done Sent Done false
54612024-01-24T19:07:47.108Z 18 Acked 1017 Read 10 Done Sent Done false
54622024-01-24T19:07:47.109Z 19 Acked 1018 Write 8 Done Sent Done false
54632024-01-24T19:07:47.109Z 20 Acked 1019 Read 1 Done Sent Done false
54642024-01-24T19:07:47.109Z 21 Acked 1020 Read 4 Done Sent Done false
54652024-01-24T19:07:47.109Z 22 Acked 1021 Write 3 Done Sent Done false
54662024-01-24T19:07:47.109Z 23 Acked 1022 Read 5 Done Sent Done false
54672024-01-24T19:07:47.110Z 24 Acked 1023 Flush 0 Done Sent Done false
54682024-01-24T19:07:47.110Z 25 Acked 1024 Flush 0 Done Sent Done false
54692024-01-24T19:07:47.110Z 26 Acked 1025 Write 7 Done Sent Done false
54702024-01-24T19:07:47.110Z 27 Acked 1026 Write 4 Done Sent Done false
54712024-01-24T19:07:47.110Z 28 Acked 1027 Read 5 Done Sent Done false
54722024-01-24T19:07:47.110Z 29 Acked 1028 Write 6 Done Sent Done false
54732024-01-24T19:07:47.110Z 30 Acked 1029 Read 1 Done Sent Done false
54742024-01-24T19:07:47.111Z STATES DS:0 DS:1 DS:2 TOTAL
54752024-01-24T19:07:47.111Z New 0 0 0 0
54762024-01-24T19:07:47.111Z Sent 0 25 0 25
54772024-01-24T19:07:47.111Z Done 30 5 30 65
54782024-01-24T19:07:47.111Z Skipped 0 0 0 0
54792024-01-24T19:07:47.111Z Error 0 0 0 0
54802024-01-24T19:07:47.111ZLast Flush: 1024 0 1024
54812024-01-24T19:07:47.111ZDownstairs last five completed:
54822024-01-24T19:07:47.111ZUpstairs last five completed: 30 29 28 27 26
54832024-01-24T19:07:47.112ZWrote out file "/tmp/test_repair_verify.data"
54842024-01-24T19:07:47.112Z
54852024-01-24T19:07:47.112ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
54862024-01-24T19:07:47.123ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54872024-01-24T19:07:47.123Z 0 000-019 49 49 49 80 80 80 F F F
54882024-01-24T19:07:47.123Z 1 020-039 50 49 50 81 80 81 T T T <---
54892024-01-24T19:07:47.124Z 2 040-059 50 47 50 81 78 81 F T F <---
54902024-01-24T19:07:47.124Z 3 060-079 47 47 47 78 78 78 F F F
54912024-01-24T19:07:47.124Z 4 080-099 47 47 47 78 78 78 F F F
54922024-01-24T19:07:47.124Z 5 100-119 47 47 47 78 78 78 F F F
54932024-01-24T19:07:47.124Z 6 120-139 50 47 50 83 78 83 F F F <---
54942024-01-24T19:07:47.124Z 7 140-159 50 41 50 83 68 83 F F F <---
54952024-01-24T19:07:47.124Z 8 160-179 50 44 50 82 75 82 F F F <---
54962024-01-24T19:07:47.124Z 9 180-199 44 44 44 75 75 75 F F F
54972024-01-24T19:07:47.125Z 10 200-219 42 42 42 69 69 69 F F F
54982024-01-24T19:07:47.125Z 11 220-239 48 48 48 79 79 79 F F F
54992024-01-24T19:07:47.125Z 12 240-259 50 49 50 83 80 83 F F F <---
55002024-01-24T19:07:47.125Z 13 260-279 49 49 49 80 80 80 F F F
55012024-01-24T19:07:47.125Z 14 280-299 50 49 50 83 80 83 F F F <---
55022024-01-24T19:07:47.125Z 15 300-319 39 39 39 65 65 65 F F F
55032024-01-24T19:07:47.125Z 16 320-339 47 47 47 78 78 78 T F T <---
55042024-01-24T19:07:47.125Z 17 340-359 47 47 47 78 78 78 F F F
55052024-01-24T19:07:47.125Z 18 360-379 47 47 47 78 78 78 F F F
55062024-01-24T19:07:47.125Z 19 380-399 47 47 47 78 78 78 F F F
55072024-01-24T19:07:47.126Z 20 400-419 43 43 43 70 70 70 T F T <---
55082024-01-24T19:07:47.126Z 21 420-439 50 49 50 81 80 81 F T F <---
55092024-01-24T19:07:47.126Z 22 440-459 42 42 42 69 69 69 F F F
55102024-01-24T19:07:47.126Z 23 460-479 50 43 50 81 70 81 F T F <---
55112024-01-24T19:07:47.126Z 24 480-499 45 45 45 76 76 76 F F F
55122024-01-24T19:07:47.126Z 25 500-519 45 45 45 76 76 76 F F F
55132024-01-24T19:07:47.126Z 26 520-539 44 44 44 72 72 72 F F F
55142024-01-24T19:07:47.126Z 27 540-559 50 47 50 83 78 83 F T F <---
55152024-01-24T19:07:47.126Z 28 560-579 44 44 44 72 72 72 F F F
55162024-01-24T19:07:47.127Z 29 580-599 49 49 49 80 80 80 F F F
55172024-01-24T19:07:47.127ZMax gen: 50, Max flush: 83
55182024-01-24T19:07:47.127ZError: Difference in extent metadata found!
55192024-01-24T19:07:47.127ZA Difference in extent metadata is expected here
55202024-01-24T19:07:47.127ZOn loop 25
55212024-01-24T19:07:47.127Z
55222024-01-24T19:07:47.127ZVerifying data now
55232024-01-24T19:07:53.167ZNo change after verify
55242024-01-24T19:07:53.168ZLoop: 25 Downstairs dump after verify (and repair):
55252024-01-24T19:07:53.257ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55262024-01-24T19:07:53.257Z 0 000-019 49 49 49 80 80 80 F F F
55272024-01-24T19:07:53.257Z 1 020-039 51 51 51 84 84 84 F F F
55282024-01-24T19:07:53.257Z 2 040-059 50 50 50 81 81 81 F F F
55292024-01-24T19:07:53.257Z 3 060-079 47 47 47 78 78 78 F F F
55302024-01-24T19:07:53.257Z 4 080-099 47 47 47 78 78 78 F F F
55312024-01-24T19:07:54.519Z 5 100-119 47 47 47 78 78 78 F F F
55322024-01-24T19:07:54.519Z 6 120-139 50 50 50 83 83 83 F F F
55332024-01-24T19:07:54.519Z 7 140-159 50 50 50 83 83 83 F F F
55342024-01-24T19:07:54.520Z 8 160-179 50 50 50 82 82 82 F F F
55352024-01-24T19:07:54.520Z 9 180-199 44 44 44 75 75 75 F F F
55362024-01-24T19:07:54.520Z 10 200-219 42 42 42 69 69 69 F F F
55372024-01-24T19:07:54.520Z 11 220-239 48 48 48 79 79 79 F F F
55382024-01-24T19:07:54.520Z 12 240-259 50 50 50 83 83 83 F F F
55392024-01-24T19:07:54.520Z 13 260-279 49 49 49 80 80 80 F F F
55402024-01-24T19:07:54.521Z 14 280-299 50 50 50 83 83 83 F F F
55412024-01-24T19:07:54.521Z 15 300-319 39 39 39 65 65 65 F F F
55422024-01-24T19:07:54.522Z 16 320-339 51 51 51 84 84 84 F F F
55432024-01-24T19:07:54.522Z 17 340-359 47 47 47 78 78 78 F F F
55442024-01-24T19:07:54.522Z 18 360-379 47 47 47 78 78 78 F F F
55452024-01-24T19:07:54.523Z 19 380-399 47 47 47 78 78 78 F F F
55462024-01-24T19:07:54.523Z 20 400-419 51 51 51 84 84 84 F F F
55472024-01-24T19:07:54.523Z 21 420-439 50 50 50 81 81 81 F F F
55482024-01-24T19:07:54.523Z 22 440-459 42 42 42 69 69 69 F F F
55492024-01-24T19:07:54.523Z 23 460-479 50 50 50 81 81 81 F F F
55502024-01-24T19:07:54.523Z 24 480-499 45 45 45 76 76 76 F F F
55512024-01-24T19:07:54.523Z 25 500-519 45 45 45 76 76 76 F F F
55522024-01-24T19:07:54.524Z 26 520-539 44 44 44 72 72 72 F F F
55532024-01-24T19:07:54.524Z 27 540-559 50 50 50 83 83 83 F F F
55542024-01-24T19:07:54.524Z 28 560-579 44 44 44 72 72 72 F F F
55552024-01-24T19:07:54.524Z 29 580-599 49 49 49 80 80 80 F F F
55562024-01-24T19:07:54.524ZMax gen: 51, Max flush: 84
55572024-01-24T19:07:54.524Z
55582024-01-24T19:07:54.524ZBegin loop 26
55592024-01-24T19:07:54.525ZDownstairs to restart: 0
55602024-01-24T19:07:54.525Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881312101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337}
55612024-01-24T19:07:54.525Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881753748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337}
55622024-01-24T19:07:54.525Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881783789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337}
55632024-01-24T19:07:54.525Z{"msg":"Crucible 60a3978c-941e-473b-bf78-5af3761ca98f has session id: 36d20963-5492-477b-894a-ad0650c22cff","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881819878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55642024-01-24T19:07:54.525Z{"msg":"Upstairs opts: Upstairs UUID: 60a3978c-941e-473b-bf78-5af3761ca98f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881879384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55652024-01-24T19:07:54.525Z{"msg":"Crucible stats registered with UUID: 60a3978c-941e-473b-bf78-5af3761ca98f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.881951543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55662024-01-24T19:07:54.526ZCrucible runtime is spawned
55672024-01-24T19:07:54.526Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882242194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337}
55682024-01-24T19:07:54.526Z{"msg":"60a3978c-941e-473b-bf78-5af3761ca98f active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882474767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55692024-01-24T19:07:54.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882538418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55702024-01-24T19:07:54.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88256631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55712024-01-24T19:07:54.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882623449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55722024-01-24T19:07:54.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882667018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55732024-01-24T19:07:54.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.8827068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55742024-01-24T19:07:54.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882749594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55752024-01-24T19:07:54.528Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88282502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55762024-01-24T19:07:54.528Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882884355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55772024-01-24T19:07:54.528Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.882987466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55782024-01-24T19:07:54.528Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.883077454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55792024-01-24T19:07:54.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.883284361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55802024-01-24T19:07:54.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.883316457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55812024-01-24T19:07:54.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88426932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55822024-01-24T19:07:54.529Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88435937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55832024-01-24T19:07:54.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.884432855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55842024-01-24T19:07:54.529Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.884514471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55852024-01-24T19:07:54.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.8853941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55862024-01-24T19:07:54.530Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88547049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55872024-01-24T19:07:54.530Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.885500628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55882024-01-24T19:07:54.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.885665974Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55892024-01-24T19:07:54.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.886521698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55902024-01-24T19:07:54.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88657022Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55912024-01-24T19:07:54.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.886595822Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55922024-01-24T19:07:54.531Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.886671525Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55932024-01-24T19:07:54.532Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88689015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55942024-01-24T19:07:54.532Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.886948629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55952024-01-24T19:07:54.532Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.886975915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55962024-01-24T19:07:54.532Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887460948Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55972024-01-24T19:07:54.532Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887526621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55982024-01-24T19:07:54.532Z{"msg":"[0]R flush_numbers[0..12]: [80, 84, 81, 78, 78, 78, 83, 83, 82, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887555765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
55992024-01-24T19:07:54.534Z{"msg":"[0]R generation[0..12]: [49, 51, 50, 47, 47, 47, 50, 50, 50, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887618864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56002024-01-24T19:07:54.534Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887653637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56012024-01-24T19:07:54.534Z{"msg":"[1]R flush_numbers[0..12]: [80, 84, 81, 78, 78, 78, 83, 83, 82, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887713709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56022024-01-24T19:07:54.534Z{"msg":"[1]R generation[0..12]: [49, 51, 50, 47, 47, 47, 50, 50, 50, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887741777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56032024-01-24T19:07:54.534ZWait for a query_work_queue command to finish before sending IO
56042024-01-24T19:07:54.534Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887804305Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56052024-01-24T19:07:54.535Z{"msg":"[2]R flush_numbers[0..12]: [80, 84, 81, 78, 78, 78, 83, 83, 82, 75, 69, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887838821Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56062024-01-24T19:07:54.535Z{"msg":"[2]R generation[0..12]: [49, 51, 50, 47, 47, 47, 50, 50, 50, 44, 42, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887912129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56072024-01-24T19:07:54.535Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887940998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56082024-01-24T19:07:54.535Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.887999983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56092024-01-24T19:07:54.535Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88803269Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56102024-01-24T19:07:54.536Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888090212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56112024-01-24T19:07:54.536Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888117403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56122024-01-24T19:07:54.536Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888176528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56132024-01-24T19:07:54.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888202205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56142024-01-24T19:07:54.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888262753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56152024-01-24T19:07:54.536Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888296079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56162024-01-24T19:07:54.536Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888353591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56172024-01-24T19:07:54.536Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.88837927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56182024-01-24T19:07:54.536Z{"msg":"60a3978c-941e-473b-bf78-5af3761ca98f is now active with session: 36d20963-5492-477b-894a-ad0650c22cff","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888447089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56192024-01-24T19:07:54.543Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888477952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337,"session_id":"36d20963-5492-477b-894a-ad0650c22cff"}
56202024-01-24T19:07:54.543Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56212024-01-24T19:07:54.543ZThe guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:51.888539213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1337}
56222024-01-24T19:07:54.543ZLoading write count information from file "/tmp/test_repair_verify.data"
56232024-01-24T19:07:54.543ZRun Repair workload
56242024-01-24T19:07:54.543Z01/30 Write block 72 len 20480 data: 5 5 5 5 6
56252024-01-24T19:07:54.543Z02/30 Write block 19 len 28672 data: 3 4 4 4 3 5 5
56262024-01-24T19:07:54.543Z03/30 Write block 292 len 24576 data: 6 7 8 6 5 3
56272024-01-24T19:07:54.543Z04/30 Write block 434 len 24576 data: 5 4 3 4 3 3
56282024-01-24T19:07:54.543Z05/30 Read block 210 len 4096
56292024-01-24T19:07:54.543Z06/30 Read block 534 len 40960
56302024-01-24T19:07:54.543Z07/30 Write block 242 len 32768 data: 5 7 7 8 10 8 7 7
56312024-01-24T19:07:54.554Z08/30 Write block 204 len 12288 data: 3 3 2
56322024-01-24T19:07:54.554Z09/30 Write block 115 len 32768 data: 3 2 2 3 6 6 7 7
56332024-01-24T19:07:54.554Z10/30 Flush
56342024-01-24T19:07:54.555Z11/30 Write block 387 len 4096 data: 4
56352024-01-24T19:07:54.555Z12/30 Write block 327 len 28672 data: 5 5 5 4 4 6 6
56362024-01-24T19:07:54.555Z13/30 Write block 147 len 4096 data: 5
56372024-01-24T19:07:54.555Z14/30 Write block 295 len 36864 data: 7 6 4 4 3 3 3 3 3
56382024-01-24T19:07:54.555Z15/30 Read block 437 len 20480
56392024-01-24T19:07:54.555Z16/30 Write block 162 len 40960 data: 6 6 5 5 4 5 5 5 5 6
56402024-01-24T19:07:54.557Z17/30 Write block 86 len 12288 data: 6 6 5
56412024-01-24T19:07:54.557Z18/30 Write block 151 len 16384 data: 7 7 7 7
56422024-01-24T19:07:54.557Z19/30 Read block 48 len 40960
56432024-01-24T19:07:54.558Z20/30 Read block 218 len 40960
56442024-01-24T19:07:54.558Z21/30 Read block 289 len 12288
56452024-01-24T19:07:54.558Z22/30 Write block 396 len 28672 data: 6 5 4 4 5 6 5
56462024-01-24T19:07:54.558Z23/30 Read block 335 len 4096
56472024-01-24T19:07:54.558Z24/30 Read block 345 len 4096
56482024-01-24T19:07:54.558Z25/30 Write block 583 len 16384 data: 5 5 4 5
56492024-01-24T19:07:54.558Z26/30 Read block 78 len 8192
56502024-01-24T19:07:54.558Z27/30 Write block 144 len 16384 data: 4 3 5 6
56512024-01-24T19:07:54.558Z28/30 Write block 358 len 16384 data: 4 5 6 7
56522024-01-24T19:07:54.558Z29/30 Read block 486 len 16384
56532024-01-24T19:07:54.558Z30/30 Read block 10 len 4096
56542024-01-24T19:07:54.558Z----------------------------------------------------------------
56552024-01-24T19:07:54.558Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56562024-01-24T19:07:54.558ZStates: Active Active Active
56572024-01-24T19:07:54.558ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56582024-01-24T19:07:54.558Z 1 Acked 1000 Write 5 Sent Done Done false
56592024-01-24T19:07:54.558Z 2 Acked 1001 Write 7 Sent Done Done false
56602024-01-24T19:07:54.558Z 3 Acked 1002 Write 6 Sent Done Done false
56612024-01-24T19:07:54.558Z 4 Acked 1003 Write 6 Sent Done Done false
56622024-01-24T19:07:54.558Z 5 Acked 1004 Read 1 Sent Done Done false
56632024-01-24T19:07:54.558Z 6 Acked 1005 Read 10 Sent Done Done false
56642024-01-24T19:07:54.559Z 7 Acked 1006 Write 8 Sent Done Done false
56652024-01-24T19:07:54.560Z 8 Acked 1007 Write 3 Sent Done Done false
56662024-01-24T19:07:54.560Z 9 Acked 1008 Write 8 Sent Done Done false
56672024-01-24T19:07:54.560Z 10 Acked 1009 Flush 0 Sent Done Done false
56682024-01-24T19:07:54.560Z 11 Acked 1010 Write 1 Sent Done Done false
56692024-01-24T19:07:54.560Z 12 Acked 1011 Write 7 Sent Done Done false
56702024-01-24T19:07:54.560Z 13 Acked 1012 Write 1 Sent Done Done false
56712024-01-24T19:07:54.560Z 14 Acked 1013 Write 9 Sent Done Done false
56722024-01-24T19:07:54.560Z 15 Acked 1014 Read 5 Sent Done Done false
56732024-01-24T19:07:54.560Z 16 Acked 1015 Write 10 Sent Done Done false
56742024-01-24T19:07:54.560Z 17 Acked 1016 Write 3 Sent Done Done false
56752024-01-24T19:07:54.560Z 18 Acked 1017 Write 4 Sent Done Done false
56762024-01-24T19:07:54.561Z 19 Acked 1018 Read 10 Sent Done Done false
56772024-01-24T19:07:54.561Z 20 Acked 1019 Read 10 Sent Done Done false
56782024-01-24T19:07:54.561Z 21 Acked 1020 Read 3 Sent Done Done false
56792024-01-24T19:07:54.561Z 22 Acked 1021 Write 7 Sent Done Done false
56802024-01-24T19:07:54.561Z 23 Acked 1022 Read 1 Sent Done Done false
56812024-01-24T19:07:54.561Z 24 Acked 1023 Read 1 Sent Done Done false
56822024-01-24T19:07:54.561Z 25 Acked 1024 Write 4 Sent Done Done false
56832024-01-24T19:07:54.561Z 26 Acked 1025 Read 2 Sent Done Done false
56842024-01-24T19:07:54.561Z 27 Acked 1026 Write 4 Sent Done Done false
56852024-01-24T19:07:54.561Z 28 Acked 1027 Write 4 Sent Done Done false
56862024-01-24T19:07:54.561Z 29 Acked 1028 Read 4 Sent Done Done false
56872024-01-24T19:07:54.561Z 30 Acked 1029 Read 1 Sent Done Done false
56882024-01-24T19:07:54.561Z STATES DS:0 DS:1 DS:2 TOTAL
56892024-01-24T19:07:54.561Z New 0 0 0 0
56902024-01-24T19:07:54.561Z Sent 30 0 0 30
56912024-01-24T19:07:54.561Z Done 0 30 30 60
56922024-01-24T19:07:54.561Z Skipped 0 0 0 0
56932024-01-24T19:07:54.561Z Error 0 0 0 0
56942024-01-24T19:07:54.562ZLast Flush: 0 1009 1009
56952024-01-24T19:07:54.562ZDownstairs last five completed:
56962024-01-24T19:07:54.562ZUpstairs last five completed: 30 29 28 27 26
56972024-01-24T19:07:54.562ZWrote out file "/tmp/test_repair_verify.data"
56982024-01-24T19:07:54.562Z
56992024-01-24T19:07:54.562ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57002024-01-24T19:07:54.562ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57012024-01-24T19:07:54.562Z 0 000-019 49 52 52 80 85 85 F F F <---
57022024-01-24T19:07:54.562Z 1 020-039 51 52 52 84 85 85 F F F <---
57032024-01-24T19:07:54.562Z 2 040-059 50 50 50 81 81 81 F F F
57042024-01-24T19:07:54.562Z 3 060-079 47 52 52 78 85 85 F F F <---
57052024-01-24T19:07:54.562Z 4 080-099 47 47 47 78 78 78 F T T <---
57062024-01-24T19:07:54.562Z 5 100-119 47 52 52 78 85 85 F F F <---
57072024-01-24T19:07:54.562Z 6 120-139 50 52 52 83 85 85 F F F <---
57082024-01-24T19:07:54.562Z 7 140-159 50 50 50 83 83 83 F T T <---
57092024-01-24T19:07:54.562Z 8 160-179 50 50 50 82 82 82 F T T <---
57102024-01-24T19:07:54.563Z 9 180-199 44 44 44 75 75 75 F F F
57112024-01-24T19:07:54.563Z 10 200-219 42 52 52 69 85 85 F F F <---
57122024-01-24T19:07:54.563Z 11 220-239 48 48 48 79 79 79 F F F
57132024-01-24T19:07:54.564Z 12 240-259 50 52 52 83 85 85 F F F <---
57142024-01-24T19:07:54.564Z 13 260-279 49 49 49 80 80 80 F F F
57152024-01-24T19:07:54.564Z 14 280-299 50 52 52 83 85 85 F T T <---
57162024-01-24T19:07:54.564Z 15 300-319 39 39 39 65 65 65 F T T <---
57172024-01-24T19:07:54.564Z 16 320-339 51 51 51 84 84 84 F T T <---
57182024-01-24T19:07:54.564Z 17 340-359 47 47 47 78 78 78 F T T <---
57192024-01-24T19:07:54.564Z 18 360-379 47 47 47 78 78 78 F T T <---
57202024-01-24T19:07:54.564Z 19 380-399 47 47 47 78 78 78 F T T <---
57212024-01-24T19:07:54.565Z 20 400-419 51 51 51 84 84 84 F T T <---
57222024-01-24T19:07:54.565Z 21 420-439 50 52 52 81 85 85 F F F <---
57232024-01-24T19:07:54.565Z 22 440-459 42 42 42 69 69 69 F F F
57242024-01-24T19:07:54.565Z 23 460-479 50 50 50 81 81 81 F F F
57252024-01-24T19:07:54.565Z 24 480-499 45 45 45 76 76 76 F F F
57262024-01-24T19:07:54.565Z 25 500-519 45 45 45 76 76 76 F F F
57272024-01-24T19:07:54.565Z 26 520-539 44 44 44 72 72 72 F F F
57282024-01-24T19:07:54.565Z 27 540-559 50 50 50 83 83 83 F F F
57292024-01-24T19:07:54.565Z 28 560-579 44 44 44 72 72 72 F F F
57302024-01-24T19:07:54.565Z 29 580-599 49 49 49 80 80 80 F T T <---
57312024-01-24T19:07:54.571ZMax gen: 52, Max flush: 85
57322024-01-24T19:07:54.571ZError: Difference in extent metadata found!
57332024-01-24T19:07:54.571ZA Difference in extent metadata is expected here
57342024-01-24T19:07:54.571ZOn loop 26
57352024-01-24T19:07:54.572Z
57362024-01-24T19:07:54.572ZVerifying data now
57372024-01-24T19:07:58.926ZNo change after verify
57382024-01-24T19:07:58.926ZLoop: 26 Downstairs dump after verify (and repair):
57392024-01-24T19:07:58.951ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57402024-01-24T19:07:58.951Z 0 000-019 52 52 52 85 85 85 F F F
57412024-01-24T19:07:58.978Z 1 020-039 52 52 52 85 85 85 F F F
57422024-01-24T19:07:58.979Z 2 040-059 50 50 50 81 81 81 F F F
57432024-01-24T19:07:58.979Z 3 060-079 52 52 52 85 85 85 F F F
57442024-01-24T19:07:58.981Z 4 080-099 53 53 53 86 86 86 F F F
57452024-01-24T19:07:58.982Z 5 100-119 52 52 52 85 85 85 F F F
57462024-01-24T19:07:58.982Z 6 120-139 52 52 52 85 85 85 F F F
57472024-01-24T19:07:58.982Z 7 140-159 53 53 53 86 86 86 F F F
57482024-01-24T19:07:58.983Z 8 160-179 53 53 53 86 86 86 F F F
57492024-01-24T19:07:58.983Z 9 180-199 44 44 44 75 75 75 F F F
57502024-01-24T19:07:58.984Z 10 200-219 52 52 52 85 85 85 F F F
57512024-01-24T19:07:58.984Z 11 220-239 48 48 48 79 79 79 F F F
57522024-01-24T19:07:58.984Z 12 240-259 52 52 52 85 85 85 F F F
57532024-01-24T19:07:58.984Z 13 260-279 49 49 49 80 80 80 F F F
57542024-01-24T19:07:58.985Z 14 280-299 53 53 53 86 86 86 F F F
57552024-01-24T19:07:58.985Z 15 300-319 53 53 53 86 86 86 F F F
57562024-01-24T19:07:58.985Z 16 320-339 53 53 53 86 86 86 F F F
57572024-01-24T19:07:58.985Z 17 340-359 53 53 53 86 86 86 F F F
57582024-01-24T19:07:58.985Z 18 360-379 53 53 53 86 86 86 F F F
57592024-01-24T19:07:58.986Z 19 380-399 53 53 53 86 86 86 F F F
57602024-01-24T19:07:58.986Z 20 400-419 53 53 53 86 86 86 F F F
57612024-01-24T19:07:58.986Z 21 420-439 52 52 52 85 85 85 F F F
57622024-01-24T19:07:58.986Z 22 440-459 42 42 42 69 69 69 F F F
57632024-01-24T19:07:58.987Z 23 460-479 50 50 50 81 81 81 F F F
57642024-01-24T19:07:58.987Z 24 480-499 45 45 45 76 76 76 F F F
57652024-01-24T19:07:58.987Z 25 500-519 45 45 45 76 76 76 F F F
57662024-01-24T19:07:58.987Z 26 520-539 44 44 44 72 72 72 F F F
57672024-01-24T19:07:58.987Z 27 540-559 50 50 50 83 83 83 F F F
57682024-01-24T19:07:58.987Z 28 560-579 44 44 44 72 72 72 F F F
57692024-01-24T19:07:58.987Z 29 580-599 53 53 53 86 86 86 F F F
57702024-01-24T19:07:58.987ZMax gen: 53, Max flush: 86
57712024-01-24T19:07:58.987Z
57722024-01-24T19:07:58.987ZBegin loop 27
57732024-01-24T19:07:58.987ZDownstairs to restart: 1
57742024-01-24T19:07:59.030Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.63855132Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345}
57752024-01-24T19:07:59.030Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.638979778Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345}
57762024-01-24T19:07:59.030Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639043298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345}
57772024-01-24T19:07:59.031Z{"msg":"Crucible c145219c-7455-4c1c-98ac-62cb527a5116 has session id: a96fe5ba-b068-4517-b3cf-e5b03474ff41","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639069455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57782024-01-24T19:07:59.031Z{"msg":"Upstairs opts: Upstairs UUID: c145219c-7455-4c1c-98ac-62cb527a5116, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639110179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57792024-01-24T19:07:59.031Z{"msg":"Crucible stats registered with UUID: c145219c-7455-4c1c-98ac-62cb527a5116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639175064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57802024-01-24T19:07:59.031ZCrucible runtime is spawned
57812024-01-24T19:07:59.031Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639717651Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345}
57822024-01-24T19:07:59.032Z{"msg":"c145219c-7455-4c1c-98ac-62cb527a5116 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639829262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57832024-01-24T19:07:59.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.639965759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57842024-01-24T19:07:59.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640024027Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57852024-01-24T19:07:59.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640101102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57862024-01-24T19:07:59.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640149185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57872024-01-24T19:07:59.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640219392Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57882024-01-24T19:07:59.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640293452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57892024-01-24T19:07:59.033Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640383394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57902024-01-24T19:07:59.033Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640463842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57912024-01-24T19:07:59.033Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640539378Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57922024-01-24T19:07:59.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640587002Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57932024-01-24T19:07:59.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.640946753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57942024-01-24T19:07:59.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.64100724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57952024-01-24T19:07:59.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.641754051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57962024-01-24T19:07:59.034Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.641828585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57972024-01-24T19:07:59.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.641964936Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57982024-01-24T19:07:59.034Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.642019534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
57992024-01-24T19:07:59.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.642790193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58002024-01-24T19:07:59.034Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.642873963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58012024-01-24T19:07:59.035Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.643068353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58022024-01-24T19:07:59.036Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.643245463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58032024-01-24T19:07:59.036Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.643781781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58042024-01-24T19:07:59.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644040981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58052024-01-24T19:07:59.036Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644118486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58062024-01-24T19:07:59.036Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.64416545Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58072024-01-24T19:07:59.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.64422693Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58082024-01-24T19:07:59.037Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644272121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58092024-01-24T19:07:59.037Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644329659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58102024-01-24T19:07:59.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644595229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58112024-01-24T19:07:59.039Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644669563Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58122024-01-24T19:07:59.040Z{"msg":"[0]R flush_numbers[0..12]: [85, 85, 81, 85, 86, 85, 85, 86, 86, 75, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644745023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58132024-01-24T19:07:59.040Z{"msg":"[0]R generation[0..12]: [52, 52, 50, 52, 53, 52, 52, 53, 53, 44, 52, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644791792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58142024-01-24T19:07:59.040Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644819285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58152024-01-24T19:07:59.040Z{"msg":"[1]R flush_numbers[0..12]: [85, 85, 81, 85, 86, 85, 85, 86, 86, 75, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644864075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58162024-01-24T19:07:59.040Z{"msg":"[1]R generation[0..12]: [52, 52, 50, 52, 53, 52, 52, 53, 53, 44, 52, 48]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644890005Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58172024-01-24T19:07:59.041Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644934825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58182024-01-24T19:07:59.041Z{"msg":"[2]R flush_numbers[0..12]: [85, 85, 81, 85, 86, 85, 85, 86, 86, 75, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.644961374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58192024-01-24T19:07:59.041Z{"msg":"[2]R generation[0..12]: [52, 52, 50, 52, 53, 52, 52, 53, 53, 44, 52, 48]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
58202024-01-24T19:07:59.041Z,"time":"2024-01-24T19:07:57.645023764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58212024-01-24T19:07:59.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645052268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58222024-01-24T19:07:59.044Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645095732Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58232024-01-24T19:07:59.044Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645121166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58242024-01-24T19:07:59.044Z{"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645177958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58252024-01-24T19:07:59.045Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.64520326Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58262024-01-24T19:07:59.045Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.64523152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58272024-01-24T19:07:59.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645281932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58282024-01-24T19:07:59.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645307445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58292024-01-24T19:07:59.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645329129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58302024-01-24T19:07:59.046Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645373004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58312024-01-24T19:07:59.046Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645397028Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58322024-01-24T19:07:59.047Z{"msg":"c145219c-7455-4c1c-98ac-62cb527a5116 is now active with session: a96fe5ba-b068-4517-b3cf-e5b03474ff41","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645418539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58332024-01-24T19:07:59.047Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645476618Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345,"session_id":"a96fe5ba-b068-4517-b3cf-e5b03474ff41"}
58342024-01-24T19:07:59.047Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:07:57.645548216Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1345}
58352024-01-24T19:07:59.047ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58362024-01-24T19:07:59.048ZLoading write count information from file "/tmp/test_repair_verify.data"
58372024-01-24T19:07:59.048ZRun Repair workload
58382024-01-24T19:07:59.048Z01/30 Write block 343 len 16384 data: 5 5 5 2
58392024-01-24T19:07:59.048Z02/30 Write block 591 len 24576 data: 6 8 7 6 5 4
58402024-01-24T19:07:59.048Z03/30 Read block 476 len 16384
58412024-01-24T19:07:59.048Z04/30 Read block 513 len 32768
58422024-01-24T19:07:59.048Z05/30 Read block 548 len 24576
58432024-01-24T19:07:59.048Z06/30 Read block 350 len 32768
58442024-01-24T19:07:59.049Z07/30 Read block 255 len 24576
58452024-01-24T19:07:59.050Z08/30 Read block 463 len 8192
58462024-01-24T19:07:59.064Z09/30 Write block 75 len 16384 data: 6 7 7 7
58472024-01-24T19:07:59.064Z10/30 Write block 136 len 24576 data: 5 5 6 7 6 6
58482024-01-24T19:07:59.064Z11/30 Read block 297 len 16384
58492024-01-24T19:07:59.064Z12/30 Write block 398 len 8192 data: 5 5
58502024-01-24T19:07:59.064Z13/30 Write block 547 len 20480 data: 6 7 6 5 5
58512024-01-24T19:07:59.064Z14/30 Flush
58522024-01-24T19:07:59.066Z15/30 Write block 234 len 4096 data: 4
58532024-01-24T19:07:59.066Z16/30 Read block 554 len 24576
58542024-01-24T19:07:59.067Z17/30 Read block 201 len 4096
58552024-01-24T19:07:59.067Z18/30 Read block 128 len 24576
58562024-01-24T19:07:59.067Z19/30 Write block 342 len 4096 data: 5
58572024-01-24T19:07:59.067Z20/30 Read block 128 len 16384
58582024-01-24T19:07:59.067Z21/30 Write block 44 len 32768 data: 6 7 5 4 4 4 6 5
58592024-01-24T19:07:59.067Z22/30 Write block 522 len 36864 data: 6 5 6 7 7 6 6 5 5
58602024-01-24T19:07:59.068Z23/30 Write block 418 len 40960 data: 6 7 5 5 7 6 5 4 4 6
58612024-01-24T19:07:59.068Z24/30 Read block 121 len 16384
58622024-01-24T19:07:59.068Z25/30 Write block 474 len 40960 data: 6 6 5 5 5 5 6 7 7 5
58632024-01-24T19:07:59.068Z26/30 Write block 380 len 4096 data: 6
58642024-01-24T19:07:59.068Z27/30 Read block 336 len 4096
58652024-01-24T19:07:59.068Z28/30 Write block 79 len 32768 data: 8 8 10 8 8 7 6 7
58662024-01-24T19:07:59.068Z29/30 Read block 127 len 16384
58672024-01-24T19:07:59.068Z30/30 Read block 476 len 4096
58682024-01-24T19:07:59.068Z----------------------------------------------------------------
58692024-01-24T19:07:59.069Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
58702024-01-24T19:07:59.069ZStates: Active Active Active
58712024-01-24T19:07:59.069ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
58722024-01-24T19:07:59.069Z 1 Acked 1000 Write 4 Done Sent Done false
58732024-01-24T19:07:59.069Z 2 Acked 1001 Write 6 Done Sent Done false
58742024-01-24T19:07:59.069Z 3 Acked 1002 Read 4 Done Sent Done false
58752024-01-24T19:07:59.069Z 4 Acked 1003 Read 8 Done Sent Done false
58762024-01-24T19:07:59.069Z 5 Acked 1004 Read 6 Done Sent Done false
58772024-01-24T19:07:59.070Z 6 Acked 1005 Read 8 Done Sent Done false
58782024-01-24T19:07:59.071Z 7 Acked 1006 Read 6 Done Sent Done false
58792024-01-24T19:07:59.071Z 8 Acked 1007 Read 2 Done Sent Done false
58802024-01-24T19:07:59.071Z 9 Acked 1008 Write 4 Done Sent Done false
58812024-01-24T19:07:59.071Z 10 Acked 1009 Write 6 Done Sent Done false
58822024-01-24T19:07:59.071Z 11 Acked 1010 Read 4 Done Sent Done false
58832024-01-24T19:07:59.071Z 12 Acked 1011 Write 2 Done Sent Done false
58842024-01-24T19:07:59.071Z 13 Acked 1012 Write 5 Done Sent Done false
58852024-01-24T19:07:59.072Z 14 Acked 1013 Flush 0 Done Sent Done false
58862024-01-24T19:07:59.072Z 15 Acked 1014 Write 1 Done Sent Done false
58872024-01-24T19:07:59.072Z 16 Acked 1015 Read 6 Done Sent Done false
58882024-01-24T19:07:59.072Z 17 Acked 1016 Read 1 Done Sent Done false
58892024-01-24T19:07:59.072Z 18 Acked 1017 Read 6 Done Sent Done false
58902024-01-24T19:07:59.072Z 19 Acked 1018 Write 1 Done Sent Done false
58912024-01-24T19:07:59.072Z 20 Acked 1019 Read 4 Done Sent Done false
58922024-01-24T19:07:59.073Z 21 Acked 1020 Write 8 Done Sent Done false
58932024-01-24T19:07:59.073Z 22 Acked 1021 Write 9 Done Sent Done false
58942024-01-24T19:07:59.073Z 23 Acked 1022 Write 10 Done Sent Done false
58952024-01-24T19:07:59.073Z 24 Acked 1023 Read 4 Done Sent Done false
58962024-01-24T19:07:59.073Z 25 Acked 1024 Write 10 Done Sent Done false
58972024-01-24T19:07:59.073Z 26 Acked 1025 Write 1 Done Sent Done false
58982024-01-24T19:07:59.074Z 27 Acked 1026 Read 1 Done Sent Done false
58992024-01-24T19:07:59.074Z 28 Acked 1027 Write 8 Done Sent Done false
59002024-01-24T19:07:59.076Z 29 Acked 1028 Read 4 Done Sent Done false
59012024-01-24T19:07:59.076Z 30 Acked 1029 Read 1 Done Sent Sent false
59022024-01-24T19:07:59.076Z STATES DS:0 DS:1 DS:2 TOTAL
59032024-01-24T19:07:59.076Z New 0 0 0 0
59042024-01-24T19:07:59.076Z Sent 0 30 1 31
59052024-01-24T19:07:59.077Z Done 30 0 29 59
59062024-01-24T19:07:59.077Z Skipped 0 0 0 0
59072024-01-24T19:07:59.077Z Error 0 0 0 0
59082024-01-24T19:07:59.077ZLast Flush: 1013 0 1013
59092024-01-24T19:07:59.077ZDownstairs last five completed:
59102024-01-24T19:07:59.077ZUpstairs last five completed: 30 29 28 27 26
59112024-01-24T19:07:59.077ZWrote out file "/tmp/test_repair_verify.data"
59122024-01-24T19:07:59.077Z
59132024-01-24T19:07:59.077ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59142024-01-24T19:07:59.095ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59152024-01-24T19:07:59.096Z 0 000-019 52 52 52 85 85 85 F F F
59162024-01-24T19:07:59.096Z 1 020-039 52 52 52 85 85 85 F F F
59172024-01-24T19:07:59.096Z 2 040-059 50 50 50 81 81 81 T F T <---
59182024-01-24T19:07:59.096Z 3 060-079 54 52 54 87 85 87 T F T <---
59192024-01-24T19:07:59.096Z 4 080-099 53 53 53 86 86 86 T F T <---
59202024-01-24T19:07:59.096Z 5 100-119 52 52 52 85 85 85 F F F
59212024-01-24T19:07:59.096Z 6 120-139 54 52 54 87 85 87 F F F <---
59222024-01-24T19:07:59.096Z 7 140-159 54 53 54 87 86 87 F F F <---
59232024-01-24T19:07:59.096Z 8 160-179 53 53 53 86 86 86 F F F
59242024-01-24T19:07:59.096Z 9 180-199 44 44 44 75 75 75 F F F
59252024-01-24T19:07:59.097Z 10 200-219 52 52 52 85 85 85 F F F
59262024-01-24T19:07:59.097Z 11 220-239 48 48 48 79 79 79 T F T <---
59272024-01-24T19:07:59.097Z 12 240-259 52 52 52 85 85 85 F F F
59282024-01-24T19:07:59.097Z 13 260-279 49 49 49 80 80 80 F F F
59292024-01-24T19:07:59.097Z 14 280-299 53 53 53 86 86 86 F F F
59302024-01-24T19:07:59.097Z 15 300-319 53 53 53 86 86 86 F F F
59312024-01-24T19:07:59.097Z 16 320-339 53 53 53 86 86 86 F F F
59322024-01-24T19:07:59.097Z 17 340-359 54 53 54 87 86 87 T F T <---
59332024-01-24T19:07:59.097Z 18 360-379 53 53 53 86 86 86 F F F
59342024-01-24T19:07:59.097Z 19 380-399 54 53 54 87 86 87 T F T <---
59352024-01-24T19:07:59.097Z 20 400-419 53 53 53 86 86 86 T F T <---
59362024-01-24T19:07:59.097Z 21 420-439 52 52 52 85 85 85 T F T <---
59372024-01-24T19:07:59.097Z 22 440-459 42 42 42 69 69 69 F F F
59382024-01-24T19:07:59.097Z 23 460-479 50 50 50 81 81 81 T F T <---
59392024-01-24T19:07:59.097Z 24 480-499 45 45 45 76 76 76 T F T <---
59402024-01-24T19:07:59.097Z 25 500-519 45 45 45 76 76 76 F F F
59412024-01-24T19:07:59.097Z 26 520-539 44 44 44 72 72 72 T F T <---
59422024-01-24T19:07:59.097Z 27 540-559 54 50 54 87 83 87 F F F <---
59432024-01-24T19:07:59.098Z 28 560-579 44 44 44 72 72 72 F F F
59442024-01-24T19:07:59.098Z 29 580-599 54 53 54 87 86 87 F F F <---
59452024-01-24T19:07:59.098ZMax gen: 54, Max flush: 87
59462024-01-24T19:07:59.098ZError: Difference in extent metadata found!
59472024-01-24T19:07:59.098ZA Difference in extent metadata is expected here
59482024-01-24T19:07:59.098ZOn loop 27
59492024-01-24T19:07:59.098Z
59502024-01-24T19:07:59.099ZVerifying data now
59512024-01-24T19:08:04.424ZNo change after verify
59522024-01-24T19:08:04.424ZLoop: 27 Downstairs dump after verify (and repair):
59532024-01-24T19:08:04.446ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59542024-01-24T19:08:04.446Z 0 000-019 52 52 52 85 85 85 F F F
59552024-01-24T19:08:04.446Z 1 020-039 52 52 52 85 85 85 F F F
59562024-01-24T19:08:04.446Z 2 040-059 55 55 55 88 88 88 F F F
59572024-01-24T19:08:04.446Z 3 060-079 55 55 55 88 88 88 F F F
59582024-01-24T19:08:04.447Z 4 080-099 55 55 55 88 88 88 F F F
59592024-01-24T19:08:04.447Z 5 100-119 52 52 52 85 85 85 F F F
59602024-01-24T19:08:04.450Z 6 120-139 54 54 54 87 87 87 F F F
59612024-01-24T19:08:04.452Z 7 140-159 54 54 54 87 87 87 F F F
59622024-01-24T19:08:04.452Z 8 160-179 53 53 53 86 86 86 F F F
59632024-01-24T19:08:04.452Z 9 180-199 44 44 44 75 75 75 F F F
59642024-01-24T19:08:04.452Z 10 200-219 52 52 52 85 85 85 F F F
59652024-01-24T19:08:04.452Z 11 220-239 55 55 55 88 88 88 F F F
59662024-01-24T19:08:04.452Z 12 240-259 52 52 52 85 85 85 F F F
59672024-01-24T19:08:04.452Z 13 260-279 49 49 49 80 80 80 F F F
59682024-01-24T19:08:04.452Z 14 280-299 53 53 53 86 86 86 F F F
59692024-01-24T19:08:04.452Z 15 300-319 53 53 53 86 86 86 F F F
59702024-01-24T19:08:04.452Z 16 320-339 53 53 53 86 86 86 F F F
59712024-01-24T19:08:04.452Z 17 340-359 55 55 55 88 88 88 F F F
59722024-01-24T19:08:04.452Z 18 360-379 53 53 53 86 86 86 F F F
59732024-01-24T19:08:04.452Z 19 380-399 55 55 55 88 88 88 F F F
59742024-01-24T19:08:04.452Z 20 400-419 55 55 55 88 88 88 F F F
59752024-01-24T19:08:04.452Z 21 420-439 55 55 55 88 88 88 F F F
59762024-01-24T19:08:04.452Z 22 440-459 42 42 42 69 69 69 F F F
59772024-01-24T19:08:04.453Z 23 460-479 55 55 55 88 88 88 F F F
59782024-01-24T19:08:04.453Z 24 480-499 55 55 55 88 88 88 F F F
59792024-01-24T19:08:04.453Z 25 500-519 45 45 45 76 76 76 F F F
59802024-01-24T19:08:04.453Z 26 520-539 55 55 55 88 88 88 F F F
59812024-01-24T19:08:04.453Z 27 540-559 54 54 54 87 87 87 F F F
59822024-01-24T19:08:04.453Z 28 560-579 44 44 44 72 72 72 F F F
59832024-01-24T19:08:04.453Z 29 580-599 54 54 54 87 87 87 F F F
59842024-01-24T19:08:04.453ZMax gen: 55, Max flush: 88
59852024-01-24T19:08:04.456Z
59862024-01-24T19:08:04.457ZBegin loop 28
59872024-01-24T19:08:04.457ZDownstairs to restart: 2
59882024-01-24T19:08:04.485Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.146160327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353}
59892024-01-24T19:08:04.485Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.146665583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353}
59902024-01-24T19:08:04.492Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.146747946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353}
59912024-01-24T19:08:04.492Z{"msg":"Crucible 6a5fb4f0-3e66-4afb-b432-35ca177dc821 has session id: 1fb59b6b-d8e6-424e-9515-94d56016812c","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.146829685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59922024-01-24T19:08:04.492Z{"msg":"Upstairs opts: Upstairs UUID: 6a5fb4f0-3e66-4afb-b432-35ca177dc821, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.146892628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59932024-01-24T19:08:04.492Z{"msg":"Crucible runtime is spawned
59942024-01-24T19:08:04.492ZCrucible stats registered with UUID: 6a5fb4f0-3e66-4afb-b432-35ca177dc821","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147066991Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59952024-01-24T19:08:04.492Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.14714483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353}
59962024-01-24T19:08:04.492Z{"msg":"6a5fb4f0-3e66-4afb-b432-35ca177dc821 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147443342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59972024-01-24T19:08:04.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147499141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59982024-01-24T19:08:04.493Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147591217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
59992024-01-24T19:08:04.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147630508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60002024-01-24T19:08:04.493Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147673908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60012024-01-24T19:08:04.496Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147742107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60022024-01-24T19:08:04.496Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147770496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60032024-01-24T19:08:04.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.147800474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60042024-01-24T19:08:04.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.148021236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60052024-01-24T19:08:04.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.148101039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60062024-01-24T19:08:04.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.148146149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60072024-01-24T19:08:04.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.148227378Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60082024-01-24T19:08:04.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.14827354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60092024-01-24T19:08:04.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.149353636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60102024-01-24T19:08:04.496Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.149427686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60112024-01-24T19:08:04.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.14945558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60122024-01-24T19:08:04.502Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.149482926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60132024-01-24T19:08:04.502Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.150500262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60142024-01-24T19:08:04.502Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15075894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60152024-01-24T19:08:04.514Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.151054144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60162024-01-24T19:08:04.514Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15112345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60172024-01-24T19:08:04.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15150678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60182024-01-24T19:08:04.515Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.151593044Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60192024-01-24T19:08:04.515Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.151622309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60202024-01-24T19:08:04.515Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.151832203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60212024-01-24T19:08:04.515Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.151941928Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60222024-01-24T19:08:04.515Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152080858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60232024-01-24T19:08:04.515Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152108526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60242024-01-24T19:08:04.515Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152776411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60252024-01-24T19:08:04.515Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152865265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60262024-01-24T19:08:04.515Z{"msg":"[0]R flush_numbers[0..12]: [85, 85, 88, 88, 88, 85, 87, 87, 86, 75, 85, 88]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15289298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60272024-01-24T19:08:04.515Z{"msg":"[0]R generation[0..12]: [52, 52, 55, 55, 55, 52, 54, 54, 53, 44, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152916686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60282024-01-24T19:08:04.515Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.152962154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60292024-01-24T19:08:04.515Z{"msg":"[1]R flush_numbers[0..12]: [85, 85, 88, 88, 88, 85, 87, 87, 86, 75, 85, 88]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15298861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60302024-01-24T19:08:04.515Z{"msg":"[1]R generation[0..12]: [52, 52, 55, 55, 55, 52, 54, 54, 53, 44, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153051052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60312024-01-24T19:08:04.516Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153078918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60322024-01-24T19:08:04.516Z{"msg":"[2]R flush_numbers[0..12]: [85, 85, 88, 88, 88, 85, 87, 87, 86, 75, 85, 88]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153102234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60332024-01-24T19:08:04.516Z{"msg":"[2]R generation[0..12]: [52, 52, 55, 55, 55, 52, 54, 54, 53, 44, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153152375Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60342024-01-24T19:08:04.516Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
60352024-01-24T19:08:04.516Z30,"time":"2024-01-24T19:08:03.153183241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60362024-01-24T19:08:04.516Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153205414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60372024-01-24T19:08:04.516Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153249117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60382024-01-24T19:08:04.516Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153283908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60392024-01-24T19:08:04.519Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.15334678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60402024-01-24T19:08:04.519Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153374758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60412024-01-24T19:08:04.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153394953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60422024-01-24T19:08:04.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153442895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60432024-01-24T19:08:04.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153481536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60442024-01-24T19:08:04.519Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153551119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60452024-01-24T19:08:04.519Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153577003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60462024-01-24T19:08:04.520Z{"msg":"6a5fb4f0-3e66-4afb-b432-35ca177dc821 is now active with session: 1fb59b6b-d8e6-424e-9515-94d56016812c","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153598878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60472024-01-24T19:08:04.520Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153620212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353,"session_id":"1fb59b6b-d8e6-424e-9515-94d56016812c"}
60482024-01-24T19:08:04.520Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:03.153641246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1353}
60492024-01-24T19:08:04.520ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60502024-01-24T19:08:04.520ZLoading write count information from file "/tmp/test_repair_verify.data"
60512024-01-24T19:08:04.520ZRun Repair workload
60522024-01-24T19:08:04.520Z01/30 Write block 581 len 40960 data: 6 5 6 6 5 6 5 5 5 4
60532024-01-24T19:08:04.520Z02/30 Write block 89 len 40960 data: 4 4 4 4 6 7 6 6 7 6
60542024-01-24T19:08:04.520Z03/30 Write block 266 len 24576 data: 4 4 3 3 3 3
60552024-01-24T19:08:04.520Z04/30 Read block 292 len 12288
60562024-01-24T19:08:04.520Z05/30 Write block 184 len 8192 data: 6 7
60572024-01-24T19:08:04.520Z06/30 Write block 167 len 32768 data: 6 6 6 6 7 6 5 6
60582024-01-24T19:08:04.520Z07/30 Write block 174 len 40960 data: 7 6 6 4 4 4 6 6 6 5
60592024-01-24T19:08:04.520Z08/30 Read block 426 len 12288
60602024-01-24T19:08:04.521Z09/30 Read block 384 len 24576
60612024-01-24T19:08:04.542Z10/30 Read block 274 len 16384
60622024-01-24T19:08:04.542Z11/30 Write block 472 len 28672 data: 8 6 7 7 6 6 6
60632024-01-24T19:08:04.542Z12/30 Write block 19 len 24576 data: 4 5 5 5 4 6
60642024-01-24T19:08:04.542Z13/30 Read block 372 len 4096
60652024-01-24T19:08:04.542Z14/30 Read block 373 len 36864
60662024-01-24T19:08:04.542Z15/30 Read block 259 len 20480
60672024-01-24T19:08:04.542Z16/30 Read block 562 len 4096
60682024-01-24T19:08:04.542Z17/30 Read block 568 len 40960
60692024-01-24T19:08:04.543Z18/30 Read block 122 len 40960
60702024-01-24T19:08:04.543Z19/30 Read block 392 len 20480
60712024-01-24T19:08:04.543Z20/30 Read block 549 len 24576
60722024-01-24T19:08:04.549Z21/30 Read block 452 len 8192
60732024-01-24T19:08:04.549Z22/30 Read block 308 len 12288
60742024-01-24T19:08:04.549Z23/30 Write block 386 len 24576 data: 4 5 4 4 3 4
60752024-01-24T19:08:04.549Z24/30 Read block 371 len 12288
60762024-01-24T19:08:04.549Z25/30 Write block 233 len 12288 data: 5 5 3
60772024-01-24T19:08:04.549Z26/30 Write block 403 len 12288 data: 4 3 5
60782024-01-24T19:08:04.550Z27/30 Write block 136 len 8192 data: 6 6
60792024-01-24T19:08:04.550Z28/30 Read block 481 len 8192
60802024-01-24T19:08:04.550Z29/30 Write block 428 len 28672 data: 7 7 6 6 6 6 6
60812024-01-24T19:08:04.550Z30/30 Read block 345 len 4096
60822024-01-24T19:08:04.550Z----------------------------------------------------------------
60832024-01-24T19:08:04.550Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
60842024-01-24T19:08:04.550ZStates: Active Active Active
60852024-01-24T19:08:04.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
60862024-01-24T19:08:04.551Z 1 Acked 1000 Write 10 Done Done Done false
60872024-01-24T19:08:04.551Z 2 Acked 1001 Write 10 Done Done Sent false
60882024-01-24T19:08:04.551Z 3 Acked 1002 Write 6 Done Done Sent false
60892024-01-24T19:08:04.551Z 4 Acked 1003 Read 3 Done Done Sent false
60902024-01-24T19:08:04.551Z 5 Acked 1004 Write 2 Done Done Sent false
60912024-01-24T19:08:04.551Z 6 Acked 1005 Write 8 Done Done Sent false
60922024-01-24T19:08:04.551Z 7 Acked 1006 Write 10 Done Done Sent false
60932024-01-24T19:08:04.551Z 8 Acked 1007 Read 3 Done Done Sent false
60942024-01-24T19:08:04.551Z 9 Acked 1008 Read 6 Done Done Sent false
60952024-01-24T19:08:04.551Z 10 Acked 1009 Read 4 Done Done Sent false
60962024-01-24T19:08:04.551Z 11 Acked 1010 Write 7 Done Done Sent false
60972024-01-24T19:08:04.551Z 12 Acked 1011 Write 6 Done Done Sent false
60982024-01-24T19:08:04.551Z 13 Acked 1012 Read 1 Done Done Sent false
60992024-01-24T19:08:04.551Z 14 Acked 1013 Read 9 Done Done Sent false
61002024-01-24T19:08:04.551Z 15 Acked 1014 Read 5 Done Done Sent false
61012024-01-24T19:08:04.551Z 16 Acked 1015 Read 1 Done Done Sent false
61022024-01-24T19:08:04.551Z 17 Acked 1016 Read 10 Done Done Sent false
61032024-01-24T19:08:04.551Z 18 Acked 1017 Read 10 Done Done Sent false
61042024-01-24T19:08:04.551Z 19 Acked 1018 Read 5 Done Done Sent false
61052024-01-24T19:08:04.551Z 20 Acked 1019 Read 6 Done Done Sent false
61062024-01-24T19:08:04.551Z 21 Acked 1020 Read 2 Done Done Sent false
61072024-01-24T19:08:04.552Z 22 Acked 1021 Read 3 Done Done Sent false
61082024-01-24T19:08:04.552Z 23 Acked 1022 Write 6 Done Done Sent false
61092024-01-24T19:08:04.552Z 24 Acked 1023 Read 3 Done Done Sent false
61102024-01-24T19:08:04.552Z 25 Acked 1024 Write 3 Done Done Sent false
61112024-01-24T19:08:04.552Z 26 Acked 1025 Write 3 Done Done Sent false
61122024-01-24T19:08:04.552Z 27 Acked 1026 Write 2 Done Done Sent false
61132024-01-24T19:08:04.552Z 28 Acked 1027 Read 2 Done Done Sent false
61142024-01-24T19:08:04.552Z 29 Acked 1028 Write 7 Done Done Sent false
61152024-01-24T19:08:04.552Z 30 Acked 1029 Read 1 Done Done Sent false
61162024-01-24T19:08:04.552Z STATES DS:0 DS:1 DS:2 TOTAL
61172024-01-24T19:08:04.552Z New 0 0 0 0
61182024-01-24T19:08:04.552Z Sent 0 0 29 29
61192024-01-24T19:08:04.552Z Done 30 30 1 61
61202024-01-24T19:08:04.552Z Skipped 0 0 0 0
61212024-01-24T19:08:04.552Z Error 0 0 0 0
61222024-01-24T19:08:04.552ZLast Flush: 0 0 0
61232024-01-24T19:08:04.553ZDownstairs last five completed:
61242024-01-24T19:08:04.553ZUpstairs last five completed: 30 29 28 27 26
61252024-01-24T19:08:04.553ZWrote out file "/tmp/test_repair_verify.data"
61262024-01-24T19:08:04.553Z
61272024-01-24T19:08:04.557ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61282024-01-24T19:08:04.585ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61292024-01-24T19:08:04.585Z 0 000-019 52 52 52 85 85 85 T T F <---
61302024-01-24T19:08:04.585Z 1 020-039 52 52 52 85 85 85 T T F <---
61312024-01-24T19:08:04.586Z 2 040-059 55 55 55 88 88 88 F F F
61322024-01-24T19:08:04.586Z 3 060-079 55 55 55 88 88 88 F F F
61332024-01-24T19:08:04.586Z 4 080-099 55 55 55 88 88 88 T T F <---
61342024-01-24T19:08:04.586Z 5 100-119 52 52 52 85 85 85 F F F
61352024-01-24T19:08:05.012Z 6 120-139 54 54 54 87 87 87 T T F <---
61362024-01-24T19:08:05.012Z 7 140-159 54 54 54 87 87 87 F F F
61372024-01-24T19:08:05.012Z 8 160-179 53 53 53 86 86 86 T T F <---
61382024-01-24T19:08:05.012Z 9 180-199 44 44 44 75 75 75 T T F <---
61392024-01-24T19:08:05.012Z 10 200-219 52 52 52 85 85 85 F F F
61402024-01-24T19:08:05.012Z 11 220-239 55 55 55 88 88 88 T T F <---
61412024-01-24T19:08:05.012Z 12 240-259 52 52 52 85 85 85 F F F
61422024-01-24T19:08:05.012Z 13 260-279 49 49 49 80 80 80 T T F <---
61432024-01-24T19:08:05.012Z 14 280-299 53 53 53 86 86 86 F F F
61442024-01-24T19:08:05.013Z 15 300-319 53 53 53 86 86 86 F F F
61452024-01-24T19:08:05.013Z 16 320-339 53 53 53 86 86 86 F F F
61462024-01-24T19:08:05.013Z 17 340-359 55 55 55 88 88 88 F F F
61472024-01-24T19:08:05.013Z 18 360-379 53 53 53 86 86 86 F F F
61482024-01-24T19:08:05.013Z 19 380-399 55 55 55 88 88 88 T T F <---
61492024-01-24T19:08:05.013Z 20 400-419 55 55 55 88 88 88 T T F <---
61502024-01-24T19:08:05.014Z 21 420-439 55 55 55 88 88 88 T T F <---
61512024-01-24T19:08:05.014Z 22 440-459 42 42 42 69 69 69 F F F
61522024-01-24T19:08:05.014Z 23 460-479 55 55 55 88 88 88 T T F <---
61532024-01-24T19:08:05.014Z 24 480-499 55 55 55 88 88 88 F F F
61542024-01-24T19:08:05.014Z 25 500-519 45 45 45 76 76 76 F F F
61552024-01-24T19:08:05.014Z 26 520-539 55 55 55 88 88 88 F F F
61562024-01-24T19:08:05.014Z 27 540-559 54 54 54 87 87 87 F F F
61572024-01-24T19:08:05.014Z 28 560-579 44 44 44 72 72 72 F F F
61582024-01-24T19:08:05.014Z 29 580-599 54 54 54 87 87 87 T T T
61592024-01-24T19:08:05.014ZMax gen: 55, Max flush: 88
61602024-01-24T19:08:05.014ZError: Difference in extent metadata found!
61612024-01-24T19:08:05.014ZA Difference in extent metadata is expected here
61622024-01-24T19:08:05.014ZOn loop 28
61632024-01-24T19:08:05.014Z
61642024-01-24T19:08:05.047ZVerifying data now
61652024-01-24T19:08:09.808ZNo change after verify
61662024-01-24T19:08:09.808ZLoop: 28 Downstairs dump after verify (and repair):
61672024-01-24T19:08:09.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61682024-01-24T19:08:09.822Z 0 000-019 56 56 56 89 89 89 F F F
61692024-01-24T19:08:09.823Z 1 020-039 56 56 56 89 89 89 F F F
61702024-01-24T19:08:09.823Z 2 040-059 55 55 55 88 88 88 F F F
61712024-01-24T19:08:09.823Z 3 060-079 55 55 55 88 88 88 F F F
61722024-01-24T19:08:09.823Z 4 080-099 56 56 56 89 89 89 F F F
61732024-01-24T19:08:09.823Z 5 100-119 52 52 52 85 85 85 F F F
61742024-01-24T19:08:09.824Z 6 120-139 56 56 56 89 89 89 F F F
61752024-01-24T19:08:09.824Z 7 140-159 54 54 54 87 87 87 F F F
61762024-01-24T19:08:09.824Z 8 160-179 56 56 56 89 89 89 F F F
61772024-01-24T19:08:09.824Z 9 180-199 56 56 56 89 89 89 F F F
61782024-01-24T19:08:09.824Z 10 200-219 52 52 52 85 85 85 F F F
61792024-01-24T19:08:09.829Z 11 220-239 56 56 56 89 89 89 F F F
61802024-01-24T19:08:09.830Z 12 240-259 52 52 52 85 85 85 F F F
61812024-01-24T19:08:09.830Z 13 260-279 56 56 56 89 89 89 F F F
61822024-01-24T19:08:09.830Z 14 280-299 53 53 53 86 86 86 F F F
61832024-01-24T19:08:09.831Z 15 300-319 53 53 53 86 86 86 F F F
61842024-01-24T19:08:09.831Z 16 320-339 53 53 53 86 86 86 F F F
61852024-01-24T19:08:09.831Z 17 340-359 55 55 55 88 88 88 F F F
61862024-01-24T19:08:09.831Z 18 360-379 53 53 53 86 86 86 F F F
61872024-01-24T19:08:09.831Z 19 380-399 56 56 56 89 89 89 F F F
61882024-01-24T19:08:09.831Z 20 400-419 56 56 56 89 89 89 F F F
61892024-01-24T19:08:09.832Z 21 420-439 56 56 56 89 89 89 F F F
61902024-01-24T19:08:09.832Z 22 440-459 42 42 42 69 69 69 F F F
61912024-01-24T19:08:09.833Z 23 460-479 56 56 56 89 89 89 F F F
61922024-01-24T19:08:09.833Z 24 480-499 55 55 55 88 88 88 F F F
61932024-01-24T19:08:09.833Z 25 500-519 45 45 45 76 76 76 F F F
61942024-01-24T19:08:09.833Z 26 520-539 55 55 55 88 88 88 F F F
61952024-01-24T19:08:09.833Z 27 540-559 54 54 54 87 87 87 F F F
61962024-01-24T19:08:09.834Z 28 560-579 44 44 44 72 72 72 F F F
61972024-01-24T19:08:09.834Z 29 580-599 56 56 56 89 89 89 F F F
61982024-01-24T19:08:09.834ZMax gen: 56, Max flush: 89
61992024-01-24T19:08:09.834Z
62002024-01-24T19:08:09.834ZBegin loop 29
62012024-01-24T19:08:09.834ZDownstairs to restart: 1
62022024-01-24T19:08:09.857Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.518375672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361}
62032024-01-24T19:08:09.858Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.518865016Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361}
62042024-01-24T19:08:09.858Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.518939784Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361}
62052024-01-24T19:08:09.874Z{"msg":"Crucible 8da6736f-af2e-4e05-9847-83ba714956e9 has session id: 9d42f729-5c58-4dc4-8d4f-08369602452d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.518982219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62062024-01-24T19:08:09.875Z{"msg":"Crucible runtime is spawned
62072024-01-24T19:08:09.875ZUpstairs opts: Upstairs UUID: 8da6736f-af2e-4e05-9847-83ba714956e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519227392Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62082024-01-24T19:08:09.875Z{"msg":"Crucible stats registered with UUID: 8da6736f-af2e-4e05-9847-83ba714956e9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519298185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62092024-01-24T19:08:09.875Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.51932552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361}
62102024-01-24T19:08:09.875Z{"msg":"8da6736f-af2e-4e05-9847-83ba714956e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519604939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62112024-01-24T19:08:09.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519664505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62122024-01-24T19:08:09.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519697914Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62132024-01-24T19:08:09.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519835202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62142024-01-24T19:08:09.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519920155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62152024-01-24T19:08:09.878Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.519965635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62162024-01-24T19:08:09.878Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520010745Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62172024-01-24T19:08:09.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520054484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62182024-01-24T19:08:09.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520098608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62192024-01-24T19:08:09.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520141658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62202024-01-24T19:08:09.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520181841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62212024-01-24T19:08:09.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52021831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62222024-01-24T19:08:09.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.520258742Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62232024-01-24T19:08:09.879Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52152621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62242024-01-24T19:08:09.879Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.521592682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62252024-01-24T19:08:09.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.521620744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62262024-01-24T19:08:09.880Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.521701684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62272024-01-24T19:08:09.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52242539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62282024-01-24T19:08:09.880Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.522530263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62292024-01-24T19:08:09.880Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.522770226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62302024-01-24T19:08:09.880Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.522973166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62312024-01-24T19:08:09.880Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52344911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62322024-01-24T19:08:09.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52370164Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62332024-01-24T19:08:09.880Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.523777303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62342024-01-24T19:08:09.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.523822083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62352024-01-24T19:08:09.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.523919515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62362024-01-24T19:08:09.881Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.523958644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62372024-01-24T19:08:09.881Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524010345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62382024-01-24T19:08:09.881Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524335649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62392024-01-24T19:08:09.881Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524393981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62402024-01-24T19:08:09.881Z{"msg":"[0]R flush_numbers[0..12]: [89, 89, 88, 88, 89, 85, 89, 87, 89, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524422011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62412024-01-24T19:08:09.881Z{"msg":"[0]R generation[0..12]: [56, 56, 55, 55, 56, 52, 56, 54, 56, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524466212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62422024-01-24T19:08:09.881Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524493697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62432024-01-24T19:08:09.882Z{"msg":"[1]R flush_numbers[0..12]: [89, 89, 88, 88, 89, 85, 89, 87, 89, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524537724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62442024-01-24T19:08:09.882Z{"msg":"[1]R generation[0..12]: [56, 56, 55, 55, 56, 52, 56, 54, 56, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524564497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62452024-01-24T19:08:09.882Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524608798Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361Wait for a query_work_queue command to finish before sending IO
62462024-01-24T19:08:09.883Z,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62472024-01-24T19:08:09.883Z{"msg":"[2]R flush_numbers[0..12]: [89, 89, 88, 88, 89, 85, 89, 87, 89, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524639589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62482024-01-24T19:08:09.883Z{"msg":"[2]R generation[0..12]: [56, 56, 55, 55, 56, 52, 56, 54, 56, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524683908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62492024-01-24T19:08:09.884Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524710969Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62502024-01-24T19:08:09.884Z{"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.52475436Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62512024-01-24T19:08:09.884Z{"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524780437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62522024-01-24T19:08:09.884Z{"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524822367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62532024-01-24T19:08:09.884Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524847544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62542024-01-24T19:08:09.884Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524910337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62552024-01-24T19:08:09.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.524936955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62562024-01-24T19:08:09.885Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525031185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62572024-01-24T19:08:09.885Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525075193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62582024-01-24T19:08:09.885Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525136724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62592024-01-24T19:08:09.885Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525177484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62602024-01-24T19:08:09.885Z{"msg":"8da6736f-af2e-4e05-9847-83ba714956e9 is now active with session: 9d42f729-5c58-4dc4-8d4f-08369602452d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525244236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62612024-01-24T19:08:09.885Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525283773Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361,"session_id":"9d42f729-5c58-4dc4-8d4f-08369602452d"}
62622024-01-24T19:08:09.885Z{"msg":"The guest has finished waiting for activation with:58"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62632024-01-24T19:08:09.885Z,"v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:08.525369179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1361}
62642024-01-24T19:08:09.885ZLoading write count information from file "/tmp/test_repair_verify.data"
62652024-01-24T19:08:09.885ZRun Repair workload
62662024-01-24T19:08:09.885Z01/30 Write block 539 len 4096 data: 4
62672024-01-24T19:08:09.885Z02/30 Write block 151 len 12288 data: 8 8 8
62682024-01-24T19:08:09.886Z03/30 Read block 40 len 40960
62692024-01-24T19:08:09.886Z04/30 Write block 506 len 28672 data: 3 5 4 4 5 6 6
62702024-01-24T19:08:09.890Z05/30 Read block 396 len 32768
62712024-01-24T19:08:09.890Z06/30 Read block 293 len 24576
62722024-01-24T19:08:09.890Z07/30 Read block 326 len 8192
62732024-01-24T19:08:09.890Z08/30 Write block 83 len 16384 data: 9 8 7 8
62742024-01-24T19:08:09.890Z09/30 Write block 161 len 8192 data: 6 7
62752024-01-24T19:08:09.890Z10/30 Read block 123 len 8192
62762024-01-24T19:08:09.890Z11/30 Read block 344 len 40960
62772024-01-24T19:08:09.890Z12/30 Read block 539 len 4096
62782024-01-24T19:08:09.890Z13/30 Write block 459 len 4096 data: 6
62792024-01-24T19:08:09.890Z14/30 Read block 219 len 40960
62802024-01-24T19:08:09.890Z15/30 Flush
62812024-01-24T19:08:09.890Z16/30 Write block 324 len 28672 data: 7 9 6 6 6 6 5
62822024-01-24T19:08:09.894Z17/30 Write block 373 len 16384 data: 6 7 7 6
62832024-01-24T19:08:09.894Z18/30 Write block 405 len 20480 data: 6 5 5 5 3
62842024-01-24T19:08:09.894Z19/30 Write block 130 len 12288 data: 6 6 6
62852024-01-24T19:08:09.894Z20/30 Read block 465 len 24576
62862024-01-24T19:08:09.897Z21/30 Flush
62872024-01-24T19:08:09.901Z22/30 Write block 268 len 24576 data: 4 4 4 4 3 5
62882024-01-24T19:08:09.901Z23/30 Write block 450 len 16384 data: 5 5 6 7
62892024-01-24T19:08:09.901Z24/30 Read block 587 len 16384
62902024-01-24T19:08:09.904Z25/30 Read block 459 len 20480
62912024-01-24T19:08:09.907Z26/30 Write block 587 len 28672 data: 6 6 6 5 7 9 8
62922024-01-24T19:08:09.907Z27/30 Write block 527 len 24576 data: 7 7 6 6 3 3
62932024-01-24T19:08:09.907Z28/30 Write block 510 len 28672 data: 6 7 7 8 7 7 5
62942024-01-24T19:08:09.914Z29/30 Read block 304 len 32768
62952024-01-24T19:08:09.914Z30/30 Write block 7 len 8192 data: 5 5
62962024-01-24T19:08:09.918Z----------------------------------------------------------------
62972024-01-24T19:08:09.919Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
62982024-01-24T19:08:09.919ZStates: Active Active Active
62992024-01-24T19:08:09.919ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63002024-01-24T19:08:09.919Z 1 Acked 1000 Write 1 Done Done Done false
63012024-01-24T19:08:09.919Z 2 Acked 1001 Write 3 Done Done Done false
63022024-01-24T19:08:09.919Z 3 Acked 1002 Read 10 Done Done Done false
63032024-01-24T19:08:09.919Z 4 Acked 1003 Write 7 Done Sent Done false
63042024-01-24T19:08:09.919Z 5 Acked 1004 Read 8 Done Sent Done false
63052024-01-24T19:08:09.919Z 6 Acked 1005 Read 6 Done Sent Done false
63062024-01-24T19:08:09.919Z 7 Acked 1006 Read 2 Done Sent Done false
63072024-01-24T19:08:09.919Z 8 Acked 1007 Write 4 Done Sent Done false
63082024-01-24T19:08:09.919Z 9 Acked 1008 Write 2 Done Sent Done false
63092024-01-24T19:08:09.919Z 10 Acked 1009 Read 2 Done Sent Done false
63102024-01-24T19:08:09.919Z 11 Acked 1010 Read 10 Done Sent Done false
63112024-01-24T19:08:09.919Z 12 Acked 1011 Read 1 Done Sent Done false
63122024-01-24T19:08:09.919Z 13 Acked 1012 Write 1 Done Sent Done false
63132024-01-24T19:08:09.920Z 14 Acked 1013 Read 10 Done Sent Done false
63142024-01-24T19:08:09.920Z 15 Acked 1014 Flush 0 Done Sent Done false
63152024-01-24T19:08:09.920Z 16 Acked 1015 Write 7 Done Sent Done false
63162024-01-24T19:08:09.920Z 17 Acked 1016 Write 4 Done Sent Done false
63172024-01-24T19:08:09.920Z 18 Acked 1017 Write 5 Done Sent Done false
63182024-01-24T19:08:09.920Z 19 Acked 1018 Write 3 Done Sent Done false
63192024-01-24T19:08:09.920Z 20 Acked 1019 Read 6 Done Sent Done false
63202024-01-24T19:08:09.920Z 21 Acked 1020 Flush 0 Done Sent Done false
63212024-01-24T19:08:09.920Z 22 Acked 1021 Write 6 Done Sent Done false
63222024-01-24T19:08:09.920Z 23 Acked 1022 Write 4 Done Sent Done false
63232024-01-24T19:08:09.920Z 24 Acked 1023 Read 4 Done Sent Done false
63242024-01-24T19:08:09.920Z 25 Acked 1024 Read 5 Done Sent Done false
63252024-01-24T19:08:09.920Z 26 Acked 1025 Write 7 Done Sent Done false
63262024-01-24T19:08:09.920Z 27 Acked 1026 Write 6 Done Sent Done false
63272024-01-24T19:08:09.920Z 28 Acked 1027 Write 7 Done Sent Done false
63282024-01-24T19:08:09.920Z 29 Acked 1028 Read 8 Done Sent Done false
63292024-01-24T19:08:09.920Z 30 Acked 1029 Write 2 Sent Sent Sent false
63302024-01-24T19:08:09.920Z STATES DS:0 DS:1 DS:2 TOTAL
63312024-01-24T19:08:09.920Z New 0 0 0 0
63322024-01-24T19:08:09.920Z Sent 1 27 1 29
63332024-01-24T19:08:09.920Z Done 29 3 29 61
63342024-01-24T19:08:09.920Z Skipped 0 0 0 0
63352024-01-24T19:08:09.920Z Error 0 0 0 0
63362024-01-24T19:08:09.921ZLast Flush: 1020 0 1020
63372024-01-24T19:08:09.921ZDownstairs last five completed:
63382024-01-24T19:08:09.921ZUpstairs last five completed: 30 29 28 27 26
63392024-01-24T19:08:09.921ZWrote out file "/tmp/test_repair_verify.data"
63402024-01-24T19:08:09.924Z
63412024-01-24T19:08:09.930ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63422024-01-24T19:08:09.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63432024-01-24T19:08:09.952Z 0 000-019 56 56 56 89 89 89 T F T <---
63442024-01-24T19:08:09.952Z 1 020-039 56 56 56 89 89 89 F F F
63452024-01-24T19:08:09.952Z 2 040-059 55 55 55 88 88 88 F F F
63462024-01-24T19:08:09.952Z 3 060-079 55 55 55 88 88 88 F F F
63472024-01-24T19:08:09.952Z 4 080-099 58 56 58 90 89 90 F F F <---
63482024-01-24T19:08:09.952Z 5 100-119 52 52 52 85 85 85 F F F
63492024-01-24T19:08:09.952Z 6 120-139 58 56 58 91 89 91 F F F <---
63502024-01-24T19:08:09.966Z 7 140-159 58 54 58 90 87 90 F T F <---
63512024-01-24T19:08:09.966Z 8 160-179 58 56 58 90 89 90 F F F <---
63522024-01-24T19:08:09.966Z 9 180-199 56 56 56 89 89 89 F F F
63532024-01-24T19:08:09.966Z 10 200-219 52 52 52 85 85 85 F F F
63542024-01-24T19:08:09.966Z 11 220-239 56 56 56 89 89 89 F F F
63552024-01-24T19:08:09.966Z 12 240-259 52 52 52 85 85 85 F F F
63562024-01-24T19:08:09.967Z 13 260-279 56 56 56 89 89 89 T F T <---
63572024-01-24T19:08:09.967Z 14 280-299 53 53 53 86 86 86 F F F
63582024-01-24T19:08:09.967Z 15 300-319 53 53 53 86 86 86 F F F
63592024-01-24T19:08:09.967Z 16 320-339 58 53 58 91 86 91 F F F <---
63602024-01-24T19:08:09.967Z 17 340-359 55 55 55 88 88 88 F F F
63612024-01-24T19:08:09.967Z 18 360-379 58 53 58 91 86 91 F F F <---
63622024-01-24T19:08:09.967Z 19 380-399 56 56 56 89 89 89 F F F
63632024-01-24T19:08:09.967Z 20 400-419 58 56 58 91 89 91 F F F <---
63642024-01-24T19:08:09.967Z 21 420-439 56 56 56 89 89 89 F F F
63652024-01-24T19:08:09.967Z 22 440-459 58 42 58 90 69 90 T F T <---
63662024-01-24T19:08:09.967Z 23 460-479 56 56 56 89 89 89 F F F
63672024-01-24T19:08:09.967Z 24 480-499 55 55 55 88 88 88 F F F
63682024-01-24T19:08:09.967Z 25 500-519 58 45 58 90 76 90 T F T <---
63692024-01-24T19:08:09.967Z 26 520-539 58 55 58 90 88 90 T T T <---
63702024-01-24T19:08:09.968Z 27 540-559 54 54 54 87 87 87 F F F
63712024-01-24T19:08:09.968Z 28 560-579 44 44 44 72 72 72 F F F
63722024-01-24T19:08:09.968Z 29 580-599 56 56 56 89 89 89 T F T <---
63732024-01-24T19:08:09.968ZMax gen: 58, Max flush: 91
63742024-01-24T19:08:09.968ZError: Difference in extent metadata found!
63752024-01-24T19:08:09.968ZA Difference in extent metadata is expected here
63762024-01-24T19:08:09.968ZOn loop 29
63772024-01-24T19:08:09.968Z
63782024-01-24T19:08:09.968ZVerifying data now
63792024-01-24T19:08:16.129ZNo change after verify
63802024-01-24T19:08:16.129ZLoop: 29 Downstairs dump after verify (and repair):
63812024-01-24T19:08:17.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63822024-01-24T19:08:17.377Z 0 000-019 59 59 59 92 92 92 F F F
63832024-01-24T19:08:17.387Z 1 020-039 56 56 56 89 89 89 F F F
63842024-01-24T19:08:17.388Z 2 040-059 55 55 55 88 88 88 F F F
63852024-01-24T19:08:17.388Z 3 060-079 55 55 55 88 88 88 F F F
63862024-01-24T19:08:17.388Z 4 080-099 58 58 58 90 90 90 F F F
63872024-01-24T19:08:17.388Z 5 100-119 52 52 52 85 85 85 F F F
63882024-01-24T19:08:17.389Z 6 120-139 58 58 58 91 91 91 F F F
63892024-01-24T19:08:17.389Z 7 140-159 58 58 58 90 90 90 F F F
63902024-01-24T19:08:17.389Z 8 160-179 58 58 58 90 90 90 F F F
63912024-01-24T19:08:17.389Z 9 180-199 56 56 56 89 89 89 F F F
63922024-01-24T19:08:17.389Z 10 200-219 52 52 52 85 85 85 F F F
63932024-01-24T19:08:17.389Z 11 220-239 56 56 56 89 89 89 F F F
63942024-01-24T19:08:17.390Z 12 240-259 52 52 52 85 85 85 F F F
63952024-01-24T19:08:17.390Z 13 260-279 59 59 59 92 92 92 F F F
63962024-01-24T19:08:17.390Z 14 280-299 53 53 53 86 86 86 F F F
63972024-01-24T19:08:17.390Z 15 300-319 53 53 53 86 86 86 F F F
63982024-01-24T19:08:17.390Z 16 320-339 58 58 58 91 91 91 F F F
63992024-01-24T19:08:17.390Z 17 340-359 55 55 55 88 88 88 F F F
64002024-01-24T19:08:17.390Z 18 360-379 58 58 58 91 91 91 F F F
64012024-01-24T19:08:17.390Z 19 380-399 56 56 56 89 89 89 F F F
64022024-01-24T19:08:17.390Z 20 400-419 58 58 58 91 91 91 F F F
64032024-01-24T19:08:17.390Z 21 420-439 56 56 56 89 89 89 F F F
64042024-01-24T19:08:17.391Z 22 440-459 59 59 59 92 92 92 F F F
64052024-01-24T19:08:17.391Z 23 460-479 56 56 56 89 89 89 F F F
64062024-01-24T19:08:17.391Z 24 480-499 55 55 55 88 88 88 F F F
64072024-01-24T19:08:17.391Z 25 500-519 59 59 59 92 92 92 F F F
64082024-01-24T19:08:17.391Z 26 520-539 59 59 59 92 92 92 F F F
64092024-01-24T19:08:17.391Z 27 540-559 54 54 54 87 87 87 F F F
64102024-01-24T19:08:17.391Z 28 560-579 44 44 44 72 72 72 F F F
64112024-01-24T19:08:17.391Z 29 580-599 59 59 59 92 92 92 F F F
64122024-01-24T19:08:17.391ZMax gen: 59, Max flush: 92
64132024-01-24T19:08:17.392Z
64142024-01-24T19:08:17.393ZBegin loop 30
64152024-01-24T19:08:17.393ZDownstairs to restart: 2
64162024-01-24T19:08:17.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.844298222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369}
64172024-01-24T19:08:17.393Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.844825626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369}
64182024-01-24T19:08:17.393Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.844956142Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369}
64192024-01-24T19:08:17.393Z{"msg":"Crucible c6dc368f-dad3-4bcd-8ea2-67c3f83f94b8 has session id: c5e6b89a-ff86-4173-af11-854bf853cce5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.844995569Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64202024-01-24T19:08:17.393Z{"msg":"Upstairs opts: Upstairs UUID: c6dc368f-dad3-4bcd-8ea2-67c3f83f94b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
64212024-01-24T19:08:17.394Z,"v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845084313Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64222024-01-24T19:08:17.394Z{"msg":"Crucible stats registered with UUID: c6dc368f-dad3-4bcd-8ea2-67c3f83f94b8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845174448Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64232024-01-24T19:08:17.394Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.84521906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369}
64242024-01-24T19:08:17.394Z{"msg":"c6dc368f-dad3-4bcd-8ea2-67c3f83f94b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845398455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64252024-01-24T19:08:17.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845463546Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64262024-01-24T19:08:17.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845498376Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64272024-01-24T19:08:17.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845639462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64282024-01-24T19:08:17.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845775323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64292024-01-24T19:08:17.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845860553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64302024-01-24T19:08:17.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.845944333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64312024-01-24T19:08:17.402Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846016216Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64322024-01-24T19:08:17.402Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846061901Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64332024-01-24T19:08:17.402Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846107206Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64342024-01-24T19:08:17.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846190332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64352024-01-24T19:08:17.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846236291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64362024-01-24T19:08:17.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.846278526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64372024-01-24T19:08:17.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847586697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64382024-01-24T19:08:17.403Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847671683Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64392024-01-24T19:08:17.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847753606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64402024-01-24T19:08:17.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847834407Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64412024-01-24T19:08:17.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847883152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64422024-01-24T19:08:17.403Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.847984974Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64432024-01-24T19:08:17.404Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.848595979Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64442024-01-24T19:08:17.404Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.848661988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64452024-01-24T19:08:17.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849102003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64462024-01-24T19:08:17.404Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849428243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64472024-01-24T19:08:17.405Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849517208Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64482024-01-24T19:08:17.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849563349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64492024-01-24T19:08:17.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849662149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64502024-01-24T19:08:17.406Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849725121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64512024-01-24T19:08:17.406Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.849752589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64522024-01-24T19:08:17.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850105744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64532024-01-24T19:08:17.407Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.85018629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64542024-01-24T19:08:17.407Z{"msg":"[0]R flush_numbers[0..12]: [92, 89, 88, 88, 90, 85, 91, 90, 90, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850217469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64552024-01-24T19:08:17.407Z{"msg":"[0]R generation[0..12]: [59, 56, 55, 55, 58, 52, 58, 58, 58, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850240797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64562024-01-24T19:08:17.408Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850286177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64572024-01-24T19:08:17.408Z{"msg":"[1]R flush_numbers[0..12]: [92, 89, 88, 88, 90, 85, 91, 90, 90, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850312953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64582024-01-24T19:08:17.408Z{"msg":"[1]R generation[0..12]: [59, 56, 55, 55, 58, 52, 58, 58, 58, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.8503367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64592024-01-24T19:08:17.408Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850381641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64602024-01-24T19:08:17.408Z{"msg":"[2]R flush_numbers[0..12]: [92, 89, 88, 88, 90, 85, 91, 90, 90, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850408524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64612024-01-24T19:08:17.408Z{"msg":"[2]R generation[0..12]: [59, 56, 55, 55, 58, 52, 58, 58, 58, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850431123Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64622024-01-24T19:08:17.408Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
64632024-01-24T19:08:17.408Zcrucible","level":30,"time":"2024-01-24T19:08:14.850479082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64642024-01-24T19:08:17.408Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850505008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64652024-01-24T19:08:17.409Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850527013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64662024-01-24T19:08:17.409Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850548135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64672024-01-24T19:08:17.409Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850569554Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64682024-01-24T19:08:17.409Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850612825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64692024-01-24T19:08:17.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850635898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64702024-01-24T19:08:17.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850657368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64712024-01-24T19:08:17.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850708909Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64722024-01-24T19:08:17.409Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850734308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64732024-01-24T19:08:17.409Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850754967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64742024-01-24T19:08:17.409Z{"msg":"c6dc368f-dad3-4bcd-8ea2-67c3f83f94b8 is now active with session: c5e6b89a-ff86-4173-af11-854bf853cce5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850776293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64752024-01-24T19:08:17.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850816523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369,"session_id":"c5e6b89a-ff86-4173-af11-854bf853cce5"}
64762024-01-24T19:08:17.410Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:14.850840301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1369}
64772024-01-24T19:08:17.410ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64782024-01-24T19:08:17.410ZLoading write count information from file "/tmp/test_repair_verify.data"
64792024-01-24T19:08:17.410ZRun Repair workload
64802024-01-24T19:08:17.410Z01/30 Write block 391 len 16384 data: 5 4 6 6
64812024-01-24T19:08:17.411Z02/30 Flush
64822024-01-24T19:08:17.411Z03/30 Write block 34 len 20480 data: 8 3 3 3 3
64832024-01-24T19:08:17.422Z04/30 Read block 501 len 24576
64842024-01-24T19:08:17.423Z05/30 Write block 463 len 40960 data: 3 2 4 4 5 5 5 4 5 9
64852024-01-24T19:08:17.423Z06/30 Read block 460 len 4096
64862024-01-24T19:08:17.423Z07/30 Write block 21 len 32768 data: 6 6 5 7 6 7 8 8
64872024-01-24T19:08:17.423Z08/30 Flush
64882024-01-24T19:08:17.423Z09/30 Write block 90 len 12288 data: 5 5 5
64892024-01-24T19:08:17.423Z10/30 Flush
64902024-01-24T19:08:17.423Z11/30 Write block 103 len 32768 data: 6 6 6 7 8 9 7 6
64912024-01-24T19:08:17.423Z12/30 Flush
64922024-01-24T19:08:17.423Z13/30 Write block 264 len 12288 data: 4 4 5
64932024-01-24T19:08:17.423Z14/30 Flush
64942024-01-24T19:08:17.424Z15/30 Write block 370 len 28672 data: 6 7 6 7 8 8 7
64952024-01-24T19:08:17.424Z16/30 Write block 460 len 24576 data: 6 5 4 4 3 5
64962024-01-24T19:08:17.424Z17/30 Write block 149 len 40960 data: 6 6 9 9 9 8 5 4 4 3
64972024-01-24T19:08:17.424Z18/30 Read block 529 len 16384
64982024-01-24T19:08:17.424Z19/30 Flush
64992024-01-24T19:08:17.424Z20/30 Write block 168 len 4096 data: 7
65002024-01-24T19:08:17.424Z21/30 Read block 309 len 20480
65012024-01-24T19:08:17.424Z22/30 Write block 406 len 12288 data: 6 6 6
65022024-01-24T19:08:17.425Z23/30 Flush
65032024-01-24T19:08:17.425Z24/30 Write block 272 len 32768 data: 4 6 4 3 3 6 7 10
65042024-01-24T19:08:17.425Z25/30 Flush
65052024-01-24T19:08:17.425Z26/30 Write block 541 len 12288 data: 5 7 7
65062024-01-24T19:08:17.425Z27/30 Write block 310 len 28672 data: 3 4 4 4 4 6 6
65072024-01-24T19:08:17.425Z28/30 Read block 574 len 20480
65082024-01-24T19:08:17.425Z29/30 Read block 559 len 24576
65092024-01-24T19:08:17.425Z30/30 Read block 200 len 32768
65102024-01-24T19:08:17.425Z----------------------------------------------------------------
65112024-01-24T19:08:17.425Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:28
65122024-01-24T19:08:17.425ZStates: Active Active Active
65132024-01-24T19:08:17.425ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65142024-01-24T19:08:17.426Z 3 Acked 1002 Write 5 Done Done Sent false
65152024-01-24T19:08:17.426Z 4 Acked 1003 Read 6 Done Done Sent false
65162024-01-24T19:08:17.426Z 5 Acked 1004 Write 10 Done Done Sent false
65172024-01-24T19:08:17.426Z 6 Acked 1005 Read 1 Done Done Sent false
65182024-01-24T19:08:17.426Z 7 Acked 1006 Write 8 Done Done Sent false
65192024-01-24T19:08:17.426Z 8 Acked 1007 Flush 0 Done Done Sent false
65202024-01-24T19:08:17.426Z 9 Acked 1008 Write 3 Done Done Sent false
65212024-01-24T19:08:17.426Z 10 Acked 1009 Flush 0 Done Done Sent false
65222024-01-24T19:08:17.426Z 11 Acked 1010 Write 8 Done Done Sent false
65232024-01-24T19:08:17.426Z 12 Acked 1011 Flush 0 Done Done Sent false
65242024-01-24T19:08:17.429Z 13 Acked 1012 Write 3 Done Done Sent false
65252024-01-24T19:08:17.429Z 14 Acked 1013 Flush 0 Done Done Sent false
65262024-01-24T19:08:17.429Z 15 Acked 1014 Write 7 Done Done Sent false
65272024-01-24T19:08:17.429Z 16 Acked 1015 Write 6 Done Done Sent false
65282024-01-24T19:08:17.430Z 17 Acked 1016 Write 10 Done Done Sent false
65292024-01-24T19:08:17.430Z 18 Acked 1017 Read 4 Done Done Sent false
65302024-01-24T19:08:17.430Z 19 Acked 1018 Flush 0 Done Done Sent false
65312024-01-24T19:08:17.430Z 20 Acked 1019 Write 1 Done Done Sent false
65322024-01-24T19:08:17.430Z 21 Acked 1020 Read 5 Done Done Sent false
65332024-01-24T19:08:17.430Z 22 Acked 1021 Write 3 Done Done Sent false
65342024-01-24T19:08:17.430Z 23 Acked 1022 Flush 0 Done Done Sent false
65352024-01-24T19:08:17.430Z 24 Acked 1023 Write 8 Done Done Sent false
65362024-01-24T19:08:17.430Z 25 Acked 1024 Flush 0 Done Done Sent false
65372024-01-24T19:08:17.430Z 26 Acked 1025 Write 3 Done Done Sent false
65382024-01-24T19:08:17.431Z 27 Acked 1026 Write 7 Done Done Sent false
65392024-01-24T19:08:17.431Z 28 Acked 1027 Read 5 Done Done Sent false
65402024-01-24T19:08:17.431Z 29 Acked 1028 Read 6 Done Done Sent false
65412024-01-24T19:08:17.431Z 30 Acked 1029 Read 8 Done Sent Sent false
65422024-01-24T19:08:17.431Z STATES DS:0 DS:1 DS:2 TOTAL
65432024-01-24T19:08:17.431Z New 0 0 0 0
65442024-01-24T19:08:17.431Z Sent 0 1 28 29
65452024-01-24T19:08:17.431Z Done 28 27 0 55
65462024-01-24T19:08:17.431Z Skipped 0 0 0 0
65472024-01-24T19:08:17.432Z Error 0 0 0 0
65482024-01-24T19:08:17.432ZLast Flush: 1024 1024 1001
65492024-01-24T19:08:17.432ZDownstairs last five completed: 1001 1000
65502024-01-24T19:08:17.432ZUpstairs last five completed: 30 29 28 27 26
65512024-01-24T19:08:17.432ZWrote out file "/tmp/test_repair_verify.data"
65522024-01-24T19:08:17.435Z
65532024-01-24T19:08:17.435ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65542024-01-24T19:08:17.435ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65552024-01-24T19:08:17.435Z 0 000-019 59 59 59 92 92 92 F F F
65562024-01-24T19:08:17.435Z 1 020-039 60 60 56 94 94 89 F F F <---
65572024-01-24T19:08:17.435Z 2 040-059 55 55 55 88 88 88 F F F
65582024-01-24T19:08:17.435Z 3 060-079 55 55 55 88 88 88 F F F
65592024-01-24T19:08:17.435Z 4 080-099 60 60 58 95 95 90 F F F <---
65602024-01-24T19:08:17.435Z 5 100-119 60 60 52 96 96 85 F F F <---
65612024-01-24T19:08:17.435Z 6 120-139 58 58 58 91 91 91 F F F
65622024-01-24T19:08:17.436Z 7 140-159 60 60 58 98 98 90 F F F <---
65632024-01-24T19:08:17.436Z 8 160-179 60 60 58 99 99 90 F F F <---
65642024-01-24T19:08:17.436Z 9 180-199 56 56 56 89 89 89 F F F
65652024-01-24T19:08:17.436Z 10 200-219 52 52 52 85 85 85 F F F
65662024-01-24T19:08:17.436Z 11 220-239 56 56 56 89 89 89 F F F
65672024-01-24T19:08:17.436Z 12 240-259 52 52 52 85 85 85 F F F
65682024-01-24T19:08:17.436Z 13 260-279 60 60 59 100 100 92 F F F <---
65692024-01-24T19:08:17.436Z 14 280-299 53 53 53 86 86 86 F F F
65702024-01-24T19:08:17.438Z 15 300-319 53 53 53 86 86 86 T T F <---
65712024-01-24T19:08:17.438Z 16 320-339 58 58 58 91 91 91 F F F
65722024-01-24T19:08:17.438Z 17 340-359 55 55 55 88 88 88 F F F
65732024-01-24T19:08:17.439Z 18 360-379 60 60 58 98 98 91 F F F <---
65742024-01-24T19:08:17.439Z 19 380-399 60 60 60 93 93 93 F F F
65752024-01-24T19:08:17.439Z 20 400-419 60 60 58 99 99 91 F F F <---
65762024-01-24T19:08:17.439Z 21 420-439 56 56 56 89 89 89 F F F
65772024-01-24T19:08:17.439Z 22 440-459 59 59 59 92 92 92 F F F
65782024-01-24T19:08:17.439Z 23 460-479 60 60 56 98 98 89 F F F <---
65792024-01-24T19:08:17.439Z 24 480-499 55 55 55 88 88 88 F F F
65802024-01-24T19:08:17.439Z 25 500-519 59 59 59 92 92 92 F F F
65812024-01-24T19:08:17.439Z 26 520-539 59 59 59 92 92 92 F F F
65822024-01-24T19:08:17.439Z 27 540-559 54 54 54 87 87 87 T T F <---
65832024-01-24T19:08:17.444Z 28 560-579 44 44 44 72 72 72 F F F
65842024-01-24T19:08:17.444Z 29 580-599 59 59 59 92 92 92 F F F
65852024-01-24T19:08:17.444ZMax gen: 60, Max flush: 100
65862024-01-24T19:08:17.445ZError: Difference in extent metadata found!
65872024-01-24T19:08:17.445ZA Difference in extent metadata is expected here
65882024-01-24T19:08:17.445ZOn loop 30
65892024-01-24T19:08:17.445Z
65902024-01-24T19:08:17.445ZVerifying data now
65912024-01-24T19:08:21.297ZNo change after verify
65922024-01-24T19:08:21.297ZLoop: 30 Downstairs dump after verify (and repair):
65932024-01-24T19:08:21.317ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65942024-01-24T19:08:21.317Z 0 000-019 59 59 59 92 92 92 F F F
65952024-01-24T19:08:21.317Z 1 020-039 60 60 60 94 94 94 F F F
65962024-01-24T19:08:21.318Z 2 040-059 55 55 55 88 88 88 F F F
65972024-01-24T19:08:21.318Z 3 060-079 55 55 55 88 88 88 F F F
65982024-01-24T19:08:21.318Z 4 080-099 60 60 60 95 95 95 F F F
65992024-01-24T19:08:21.318Z 5 100-119 60 60 60 96 96 96 F F F
66002024-01-24T19:08:21.318Z 6 120-139 58 58 58 91 91 91 F F F
66012024-01-24T19:08:21.319Z 7 140-159 60 60 60 98 98 98 F F F
66022024-01-24T19:08:21.319Z 8 160-179 60 60 60 99 99 99 F F F
66032024-01-24T19:08:21.728Z 9 180-199 56 56 56 89 89 89 F F F
66042024-01-24T19:08:21.729Z 10 200-219 52 52 52 85 85 85 F F F
66052024-01-24T19:08:21.729Z 11 220-239 56 56 56 89 89 89 F F F
66062024-01-24T19:08:21.729Z 12 240-259 52 52 52 85 85 85 F F F
66072024-01-24T19:08:21.729Z 13 260-279 60 60 60 100 100 100 F F F
66082024-01-24T19:08:21.729Z 14 280-299 53 53 53 86 86 86 F F F
66092024-01-24T19:08:21.730Z 15 300-319 61 61 61 101 101 101 F F F
66102024-01-24T19:08:21.730Z 16 320-339 58 58 58 91 91 91 F F F
66112024-01-24T19:08:21.730Z 17 340-359 55 55 55 88 88 88 F F F
66122024-01-24T19:08:21.730Z 18 360-379 60 60 60 98 98 98 F F F
66132024-01-24T19:08:21.731Z 19 380-399 60 60 60 93 93 93 F F F
66142024-01-24T19:08:21.731Z 20 400-419 60 60 60 99 99 99 F F F
66152024-01-24T19:08:21.731Z 21 420-439 56 56 56 89 89 89 F F F
66162024-01-24T19:08:21.731Z 22 440-459 59 59 59 92 92 92 F F F
66172024-01-24T19:08:21.731Z 23 460-479 60 60 60 98 98 98 F F F
66182024-01-24T19:08:21.732Z 24 480-499 55 55 55 88 88 88 F F F
66192024-01-24T19:08:21.732Z 25 500-519 59 59 59 92 92 92 F F F
66202024-01-24T19:08:21.732Z 26 520-539 59 59 59 92 92 92 F F F
66212024-01-24T19:08:21.733Z 27 540-559 61 61 61 101 101 101 F F F
66222024-01-24T19:08:21.733Z 28 560-579 44 44 44 72 72 72 F F F
66232024-01-24T19:08:21.733Z 29 580-599 59 59 59 92 92 92 F F F
66242024-01-24T19:08:21.733ZMax gen: 61, Max flush: 101
66252024-01-24T19:08:21.733Z
66262024-01-24T19:08:21.734ZBegin loop 31
66272024-01-24T19:08:21.734ZDownstairs to restart: 2
66282024-01-24T19:08:21.746Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.012395652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377}
66292024-01-24T19:08:21.747Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.012849397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377}
66302024-01-24T19:08:21.747Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.01292315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377}
66312024-01-24T19:08:21.747Z{"msg":"Crucible fd98d2b7-e2e4-43b6-bfaf-3f2511d44f6b has session id: 18aa3531-470b-43d8-9ec9-9c2533483bdf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.012962173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66322024-01-24T19:08:21.747Z{"msg":"Upstairs opts: Upstairs UUID: fd98d2b7-e2e4-43b6-bfaf-3f2511d44f6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
66332024-01-24T19:08:21.747Z,"time":"2024-01-24T19:08:20.01301801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66342024-01-24T19:08:21.747Z{"msg":"Crucible stats registered with UUID: fd98d2b7-e2e4-43b6-bfaf-3f2511d44f6b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013282704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66352024-01-24T19:08:21.748Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013351256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377}
66362024-01-24T19:08:21.748Z{"msg":"fd98d2b7-e2e4-43b6-bfaf-3f2511d44f6b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013571251Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66372024-01-24T19:08:21.748Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013633454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66382024-01-24T19:08:21.749Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013662191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66392024-01-24T19:08:21.749Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.0136863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66402024-01-24T19:08:21.749Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013714887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66412024-01-24T19:08:21.749Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013754237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66422024-01-24T19:08:21.749Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013823687Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66432024-01-24T19:08:21.750Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.013947649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66442024-01-24T19:08:21.750Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.01411185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66452024-01-24T19:08:21.750Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.014255942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66462024-01-24T19:08:21.750Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.014347176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66472024-01-24T19:08:21.750Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.014383526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66482024-01-24T19:08:21.752Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.014461438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66492024-01-24T19:08:21.752Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.015653707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66502024-01-24T19:08:21.753Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.015716756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66512024-01-24T19:08:21.753Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.01595115Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66522024-01-24T19:08:21.753Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.016046907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66532024-01-24T19:08:21.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.016117082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66542024-01-24T19:08:21.754Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.016146046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66552024-01-24T19:08:21.755Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.016802246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66562024-01-24T19:08:21.755Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.0172067Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66572024-01-24T19:08:21.755Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.017388908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66582024-01-24T19:08:21.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.017704075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66592024-01-24T19:08:21.756Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.017784839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66602024-01-24T19:08:21.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.017820777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66612024-01-24T19:08:21.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.017992625Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66622024-01-24T19:08:21.757Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.01807262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66632024-01-24T19:08:21.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018103513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66642024-01-24T19:08:21.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018309922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66652024-01-24T19:08:21.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018414935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66662024-01-24T19:08:21.758Z{"msg":"[0]R flush_numbers[0..12]: [92, 94, 88, 88, 95, 96, 91, 98, 99, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018463857Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66672024-01-24T19:08:21.758Z{"msg":"[0]R generation[0..12]: [59, 60, 55, 55, 60, 60, 58, 60, 60, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018537587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66682024-01-24T19:08:21.760Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018612975Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66692024-01-24T19:08:21.760Z{"msg":"[1]R flush_numbers[0..12]: [92, 94, 88, 88, 95, 96, 91, 98, 99, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018688894Z","hostname":"Wait for a query_work_queue command to finish before sending IO
66702024-01-24T19:08:21.760Zip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66712024-01-24T19:08:21.760Z{"msg":"[1]R generation[0..12]: [59, 60, 55, 55, 60, 60, 58, 60, 60, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018834628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66722024-01-24T19:08:21.760Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.01891558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66732024-01-24T19:08:21.760Z{"msg":"[2]R flush_numbers[0..12]: [92, 94, 88, 88, 95, 96, 91, 98, 99, 89, 85, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.018990474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66742024-01-24T19:08:21.760Z{"msg":"[2]R generation[0..12]: [59, 60, 55, 55, 60, 60, 58, 60, 60, 56, 52, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019071311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66752024-01-24T19:08:21.761Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019149622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66762024-01-24T19:08:21.761Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019178907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66772024-01-24T19:08:21.761Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019224209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66782024-01-24T19:08:21.762Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019249643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66792024-01-24T19:08:21.762Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019271628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66802024-01-24T19:08:21.762Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019314672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66812024-01-24T19:08:21.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019338268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66822024-01-24T19:08:21.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019360598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66832024-01-24T19:08:21.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019403539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66842024-01-24T19:08:21.763Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019429625Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66852024-01-24T19:08:21.763Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019450684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66862024-01-24T19:08:21.763Z{"msg":"fd98d2b7-e2e4-43b6-bfaf-3f2511d44f6b is now active with session: 18aa3531-470b-43d8-9ec9-9c2533483bdf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019493734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66872024-01-24T19:08:21.763Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019516767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377,"session_id":"18aa3531-470b-43d8-9ec9-9c2533483bdf"}
66882024-01-24T19:08:21.775Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:20.019537585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1377}
66892024-01-24T19:08:21.776ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
66902024-01-24T19:08:21.776ZLoading write count information from file "/tmp/test_repair_verify.data"
66912024-01-24T19:08:21.776ZRun Repair workload
66922024-01-24T19:08:21.776Z01/30 Write block 8 len 4096 data: 6
66932024-01-24T19:08:21.776Z02/30 Flush
66942024-01-24T19:08:21.776Z03/30 Write block 432 len 24576 data: 7 7 7 5 4 5
66952024-01-24T19:08:21.776Z04/30 Read block 327 len 28672
66962024-01-24T19:08:21.776Z05/30 Write block 492 len 4096 data: 4
66972024-01-24T19:08:21.776Z06/30 Write block 430 len 16384 data: 7 7 8 8
66982024-01-24T19:08:21.776Z07/30 Read block 270 len 32768
66992024-01-24T19:08:21.776Z08/30 Write block 265 len 32768 data: 5 6 5 5 5 5 5 5
67002024-01-24T19:08:21.776Z09/30 Write block 494 len 16384 data: 4 4 6 5
67012024-01-24T19:08:21.776Z10/30 Read block 525 len 16384
67022024-01-24T19:08:21.776Z11/30 Write block 517 len 32768 data: 7 8 8 7 7 7 6 7
67032024-01-24T19:08:21.776Z12/30 Write block 493 len 36864 data: 4 5 5 7 6 6 5 4 4
67042024-01-24T19:08:21.776Z13/30 Write block 255 len 40960 data: 5 5 4 5 5 5 4 3 2 5
67052024-01-24T19:08:21.776Z14/30 Read block 575 len 32768
67062024-01-24T19:08:21.776Z15/30 Write block 506 len 36864 data: 4 6 5 5 7 8 8 9 8
67072024-01-24T19:08:21.776Z16/30 Write block 72 len 12288 data: 6 6 6
67082024-01-24T19:08:21.776Z17/30 Write block 128 len 40960 data: 5 6 7 7 7 4 5 5 7 7
67092024-01-24T19:08:21.776Z18/30 Read block 70 len 28672
67102024-01-24T19:08:21.777Z19/30 Read block 227 len 32768
67112024-01-24T19:08:21.777Z20/30 Write block 204 len 40960 data: 4 4 3 3 3 4 6 5 5 5
67122024-01-24T19:08:21.777Z21/30 Read block 200 len 8192
67132024-01-24T19:08:21.777Z22/30 Read block 515 len 36864
67142024-01-24T19:08:21.777Z23/30 Write block 427 len 12288 data: 7 8 8
67152024-01-24T19:08:21.777Z24/30 Write block 163 len 28672 data: 7 6 6 5 7 8 7
67162024-01-24T19:08:21.777Z25/30 Read block 519 len 40960
67172024-01-24T19:08:21.777Z26/30 Flush
67182024-01-24T19:08:21.777Z27/30 Write block 112 len 12288 data: 5 4 2
67192024-01-24T19:08:21.777Z28/30 Read block 552 len 20480
67202024-01-24T19:08:21.777Z29/30 Read block 64 len 20480
67212024-01-24T19:08:21.778Z30/30 Write block 134 len 16384 data: 6 6 8 8
67222024-01-24T19:08:21.778Z----------------------------------------------------------------
67232024-01-24T19:08:21.778Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67242024-01-24T19:08:21.778ZStates: Active Active Active
67252024-01-24T19:08:21.778ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67262024-01-24T19:08:21.778Z 1 Acked 1000 Write 1 Done Done Sent false
67272024-01-24T19:08:21.778Z 2 Acked 1001 Flush 0 Done Done Sent false
67282024-01-24T19:08:21.800Z 3 Acked 1002 Write 6 Done Done Sent false
67292024-01-24T19:08:21.801Z 4 Acked 1003 Read 7 Done Done Sent false
67302024-01-24T19:08:21.801Z 5 Acked 1004 Write 1 Done Done Sent false
67312024-01-24T19:08:21.801Z 6 Acked 1005 Write 4 Done Done Sent false
67322024-01-24T19:08:21.801Z 7 Acked 1006 Read 8 Done Done Sent false
67332024-01-24T19:08:21.801Z 8 Acked 1007 Write 8 Done Done Sent false
67342024-01-24T19:08:21.801Z 9 Acked 1008 Write 4 Done Done Sent false
67352024-01-24T19:08:21.802Z 10 Acked 1009 Read 4 Done Done Sent false
67362024-01-24T19:08:21.802Z 11 Acked 1010 Write 8 Done Done Sent false
67372024-01-24T19:08:21.802Z 12 Acked 1011 Write 9 Done Done Sent false
67382024-01-24T19:08:21.802Z 13 Acked 1012 Write 10 Done Done Sent false
67392024-01-24T19:08:21.802Z 14 Acked 1013 Read 8 Done Done Sent false
67402024-01-24T19:08:21.802Z 15 Acked 1014 Write 9 Done Done Sent false
67412024-01-24T19:08:21.802Z 16 Acked 1015 Write 3 Done Done Sent false
67422024-01-24T19:08:21.802Z 17 Acked 1016 Write 10 Done Done Sent false
67432024-01-24T19:08:21.802Z 18 Acked 1017 Read 7 Done Done Sent false
67442024-01-24T19:08:21.802Z 19 Acked 1018 Read 8 Done Done Sent false
67452024-01-24T19:08:21.802Z 20 Acked 1019 Write 10 Done Done Sent false
67462024-01-24T19:08:21.802Z 21 Acked 1020 Read 2 Done Done Sent false
67472024-01-24T19:08:21.802Z 22 Acked 1021 Read 9 Done Done Sent false
67482024-01-24T19:08:21.802Z 23 Acked 1022 Write 3 Done Done Sent false
67492024-01-24T19:08:21.802Z 24 Acked 1023 Write 7 Done Done Sent false
67502024-01-24T19:08:21.802Z 25 Acked 1024 Read 10 Done Done Sent false
67512024-01-24T19:08:21.802Z 26 Acked 1025 Flush 0 Done Done Sent false
67522024-01-24T19:08:21.803Z 27 Acked 1026 Write 3 Done Done Sent false
67532024-01-24T19:08:21.803Z 28 Acked 1027 Read 5 Done Done Sent false
67542024-01-24T19:08:21.803Z 29 Acked 1028 Read 5 Done Done Sent false
67552024-01-24T19:08:21.804Z 30 Acked 1029 Write 4 Sent Sent Sent false
67562024-01-24T19:08:21.804Z STATES DS:0 DS:1 DS:2 TOTAL
67572024-01-24T19:08:21.804Z New 0 0 0 0
67582024-01-24T19:08:21.805Z Sent 1 1 30 32
67592024-01-24T19:08:21.805Z Done 29 29 0 58
67602024-01-24T19:08:21.805Z Skipped 0 0 0 0
67612024-01-24T19:08:21.805Z Error 0 0 0 0
67622024-01-24T19:08:21.805ZLast Flush: 1025 1025 0
67632024-01-24T19:08:21.805ZDownstairs last five completed:
67642024-01-24T19:08:21.805ZUpstairs last five completed: 30 29 28 27 26
67652024-01-24T19:08:21.805ZWrote out file "/tmp/test_repair_verify.data"
67662024-01-24T19:08:21.805Z
67672024-01-24T19:08:21.805ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67682024-01-24T19:08:21.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67692024-01-24T19:08:21.805Z 0 000-019 62 62 59 102 102 92 F F F <---
67702024-01-24T19:08:21.805Z 1 020-039 60 60 60 94 94 94 F F F
67712024-01-24T19:08:21.805Z 2 040-059 55 55 55 88 88 88 F F F
67722024-01-24T19:08:21.805Z 3 060-079 62 62 55 103 103 88 F F F <---
67732024-01-24T19:08:21.805Z 4 080-099 60 60 60 95 95 95 F F F
67742024-01-24T19:08:21.805Z 5 100-119 60 60 60 96 96 96 T T F <---
67752024-01-24T19:08:21.806Z 6 120-139 62 62 58 103 103 91 T T F <---
67762024-01-24T19:08:21.806Z 7 140-159 60 60 60 98 98 98 F F F
67772024-01-24T19:08:21.806Z 8 160-179 62 62 60 103 103 99 F F F <---
67782024-01-24T19:08:21.806Z 9 180-199 56 56 56 89 89 89 F F F
67792024-01-24T19:08:21.806Z 10 200-219 62 62 52 103 103 85 F F F <---
67802024-01-24T19:08:21.806Z 11 220-239 56 56 56 89 89 89 F F F
67812024-01-24T19:08:21.806Z 12 240-259 62 62 52 103 103 85 F F F <---
67822024-01-24T19:08:21.806Z 13 260-279 62 62 60 103 103 100 F F F <---
67832024-01-24T19:08:21.806Z 14 280-299 53 53 53 86 86 86 F F F
67842024-01-24T19:08:21.806Z 15 300-319 61 61 61 101 101 101 F F F
67852024-01-24T19:08:21.806Z 16 320-339 58 58 58 91 91 91 F F F
67862024-01-24T19:08:21.806Z 17 340-359 55 55 55 88 88 88 F F F
67872024-01-24T19:08:21.806Z 18 360-379 60 60 60 98 98 98 F F F
67882024-01-24T19:08:21.806Z 19 380-399 60 60 60 93 93 93 F F F
67892024-01-24T19:08:21.806Z 20 400-419 60 60 60 99 99 99 F F F
67902024-01-24T19:08:21.806Z 21 420-439 62 62 56 103 103 89 F F F <---
67912024-01-24T19:08:21.807Z 22 440-459 59 59 59 92 92 92 F F F
67922024-01-24T19:08:21.807Z 23 460-479 60 60 60 98 98 98 F F F
67932024-01-24T19:08:21.807Z 24 480-499 62 62 55 103 103 88 F F F <---
67942024-01-24T19:08:21.807Z 25 500-519 62 62 59 103 103 92 F F F <---
67952024-01-24T19:08:21.807Z 26 520-539 62 62 59 103 103 92 F F F <---
67962024-01-24T19:08:21.807Z 27 540-559 61 61 61 101 101 101 F F F
67972024-01-24T19:08:21.807Z 28 560-579 44 44 44 72 72 72 F F F
67982024-01-24T19:08:21.807Z 29 580-599 59 59 59 92 92 92 F F F
67992024-01-24T19:08:21.807ZMax gen: 62, Max flush: 103
68002024-01-24T19:08:21.807ZError: Difference in extent metadata found!
68012024-01-24T19:08:21.807ZA Difference in extent metadata is expected here
68022024-01-24T19:08:21.807ZOn loop 31
68032024-01-24T19:08:21.807Z
68042024-01-24T19:08:21.808ZVerifying data now
68052024-01-24T19:08:26.536ZNo change after verify
68062024-01-24T19:08:26.536ZLoop: 31 Downstairs dump after verify (and repair):
68072024-01-24T19:08:26.563ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68082024-01-24T19:08:26.564Z 0 000-019 62 62 62 102 102 102 F F F
68092024-01-24T19:08:26.585Z 1 020-039 60 60 60 94 94 94 F F F
68102024-01-24T19:08:26.585Z 2 040-059 55 55 55 88 88 88 F F F
68112024-01-24T19:08:26.586Z 3 060-079 62 62 62 103 103 103 F F F
68122024-01-24T19:08:26.586Z 4 080-099 60 60 60 95 95 95 F F F
68132024-01-24T19:08:26.586Z 5 100-119 63 63 63 104 104 104 F F F
68142024-01-24T19:08:26.586Z 6 120-139 63 63 63 104 104 104 F F F
68152024-01-24T19:08:26.586Z 7 140-159 60 60 60 98 98 98 F F F
68162024-01-24T19:08:26.586Z 8 160-179 62 62 62 103 103 103 F F F
68172024-01-24T19:08:26.586Z 9 180-199 56 56 56 89 89 89 F F F
68182024-01-24T19:08:26.587Z 10 200-219 62 62 62 103 103 103 F F F
68192024-01-24T19:08:26.587Z 11 220-239 56 56 56 89 89 89 F F F
68202024-01-24T19:08:26.587Z 12 240-259 62 62 62 103 103 103 F F F
68212024-01-24T19:08:26.587Z 13 260-279 62 62 62 103 103 103 F F F
68222024-01-24T19:08:26.587Z 14 280-299 53 53 53 86 86 86 F F F
68232024-01-24T19:08:26.588Z 15 300-319 61 61 61 101 101 101 F F F
68242024-01-24T19:08:26.588Z 16 320-339 58 58 58 91 91 91 F F F
68252024-01-24T19:08:26.588Z 17 340-359 55 55 55 88 88 88 F F F
68262024-01-24T19:08:26.588Z 18 360-379 60 60 60 98 98 98 F F F
68272024-01-24T19:08:26.588Z 19 380-399 60 60 60 93 93 93 F F F
68282024-01-24T19:08:26.588Z 20 400-419 60 60 60 99 99 99 F F F
68292024-01-24T19:08:26.588Z 21 420-439 62 62 62 103 103 103 F F F
68302024-01-24T19:08:26.588Z 22 440-459 59 59 59 92 92 92 F F F
68312024-01-24T19:08:26.589Z 23 460-479 60 60 60 98 98 98 F F F
68322024-01-24T19:08:26.589Z 24 480-499 62 62 62 103 103 103 F F F
68332024-01-24T19:08:26.589Z 25 500-519 62 62 62 103 103 103 F F F
68342024-01-24T19:08:26.589Z 26 520-539 62 62 62 103 103 103 F F F
68352024-01-24T19:08:26.589Z 27 540-559 61 61 61 101 101 101 F F F
68362024-01-24T19:08:26.589Z 28 560-579 44 44 44 72 72 72 F F F
68372024-01-24T19:08:26.590Z 29 580-599 59 59 59 92 92 92 F F F
68382024-01-24T19:08:26.590ZMax gen: 63, Max flush: 104
68392024-01-24T19:08:26.590Z
68402024-01-24T19:08:26.590ZBegin loop 32
68412024-01-24T19:08:26.592ZDownstairs to restart: 1
68422024-01-24T19:08:26.624Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
68432024-01-24T19:08:26.624Z,"time":"2024-01-24T19:08:25.250179559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385}
68442024-01-24T19:08:26.624Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250655557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385}
68452024-01-24T19:08:26.624Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25073149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385}
68462024-01-24T19:08:26.625Z{"msg":"Crucible 1b178c59-9c6e-495b-b882-5fc6c8b3b8b2 has session id: 31e5be5b-0916-45ac-8208-fa8c001d5aca","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250759969Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68472024-01-24T19:08:26.625Z{"msg":"Upstairs opts: Upstairs UUID: 1b178c59-9c6e-495b-b882-5fc6c8b3b8b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250811411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68482024-01-24T19:08:26.625Z{"msg":"Crucible stats registered with UUID: 1b178c59-9c6e-495b-b882-5fc6c8b3b8b2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250844961Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68492024-01-24T19:08:26.625Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250902115Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385}
68502024-01-24T19:08:26.625Z{"msg":"1b178c59-9c6e-495b-b882-5fc6c8b3b8b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.250997654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68512024-01-24T19:08:26.625Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25108272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68522024-01-24T19:08:26.626Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251130214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68532024-01-24T19:08:26.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251171127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68542024-01-24T19:08:26.627Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251250591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68552024-01-24T19:08:26.635Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251328127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68562024-01-24T19:08:26.635Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251369594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68572024-01-24T19:08:26.636Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251409341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68582024-01-24T19:08:26.636Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251453694Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68592024-01-24T19:08:26.636Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25149651Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68602024-01-24T19:08:26.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251571953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68612024-01-24T19:08:26.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251619744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68622024-01-24T19:08:26.638Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.251707797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68632024-01-24T19:08:26.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.252926496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68642024-01-24T19:08:26.639Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.252992752Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68652024-01-24T19:08:26.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.253144975Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68662024-01-24T19:08:26.639Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.253300598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68672024-01-24T19:08:26.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.253487176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68682024-01-24T19:08:26.640Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.253552186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68692024-01-24T19:08:26.640Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25415946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68702024-01-24T19:08:26.640Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25428745Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68712024-01-24T19:08:26.640Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.254769653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68722024-01-24T19:08:26.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255053987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68732024-01-24T19:08:26.641Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255143015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68742024-01-24T19:08:26.641Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255222244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68752024-01-24T19:08:26.641Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255295165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68762024-01-24T19:08:26.641Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255339246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68772024-01-24T19:08:26.642Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255411464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68782024-01-24T19:08:26.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255836678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68792024-01-24T19:08:26.642Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255899627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68802024-01-24T19:08:26.642Z{"msg":"[0]R flush_numbers[0..12]: [102, 94, 88, 103, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255929001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68812024-01-24T19:08:26.642Z{"msg":"[0]R generation[0..12]: [62, 60, 55, 62, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.255964783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68822024-01-24T19:08:26.642Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256002845Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68832024-01-24T19:08:26.643Z{"msg":"[1]R flush_numbers[0..12]: [102, 94, 88, 103, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256062433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68842024-01-24T19:08:26.643Z{"msg":"[1]R generation[0..12]: [62, 60, 55, 62, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256090291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68852024-01-24T19:08:26.643Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
68862024-01-24T19:08:26.643Z,"time":"2024-01-24T19:08:25.256149083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68872024-01-24T19:08:26.643Z{"msg":"[2]R flush_numbers[0..12]: [102, 94, 88, 103, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256217234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68882024-01-24T19:08:26.644Z{"msg":"[2]R generation[0..12]: [62, 60, 55, 62, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256302509Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68892024-01-24T19:08:26.644Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256373455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68902024-01-24T19:08:26.644Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256402553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68912024-01-24T19:08:26.644Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256424636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68922024-01-24T19:08:26.645Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256445775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68932024-01-24T19:08:26.646Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256471384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68942024-01-24T19:08:26.646Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.25654468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68952024-01-24T19:08:26.646Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256574896Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68962024-01-24T19:08:26.646Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256636768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68972024-01-24T19:08:26.646Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256679582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68982024-01-24T19:08:26.647Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.2567315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
68992024-01-24T19:08:26.647Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256770136Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
69002024-01-24T19:08:26.647Z{"msg":"1b178c59-9c6e-495b-b882-5fc6c8b3b8b2 is now active with session: 31e5be5b-0916-45ac-8208-fa8c001d5aca","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256834296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
69012024-01-24T19:08:26.647Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256859428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385,"session_id":"31e5be5b-0916-45ac-8208-fa8c001d5aca"}
69022024-01-24T19:08:26.647ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69032024-01-24T19:08:26.647Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:25.256922868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1385}
69042024-01-24T19:08:26.648ZLoading write count information from file "/tmp/test_repair_verify.data"
69052024-01-24T19:08:26.648ZRun Repair workload
69062024-01-24T19:08:26.648Z01/30 Flush
69072024-01-24T19:08:26.648Z02/30 Write block 316 len 24576 data: 7 7 7 7 7 7
69082024-01-24T19:08:26.648Z03/30 Read block 527 len 36864
69092024-01-24T19:08:26.648Z04/30 Write block 521 len 8192 data: 8 8
69102024-01-24T19:08:26.649Z05/30 Read block 80 len 40960
69112024-01-24T19:08:26.649Z06/30 Read block 152 len 16384
69122024-01-24T19:08:26.660Z07/30 Flush
69132024-01-24T19:08:26.660Z08/30 Write block 412 len 12288 data: 3 4 3
69142024-01-24T19:08:26.660Z09/30 Write block 60 len 24576 data: 5 4 3 4 5 4
69152024-01-24T19:08:26.660Z10/30 Write block 364 len 28672 data: 7 6 6 6 6 6 7
69162024-01-24T19:08:26.660Z11/30 Read block 521 len 4096
69172024-01-24T19:08:26.661Z12/30 Flush
69182024-01-24T19:08:26.661Z13/30 Write block 561 len 8192 data: 4 4
69192024-01-24T19:08:26.661Z14/30 Flush
69202024-01-24T19:08:26.661Z15/30 Flush
69212024-01-24T19:08:26.661Z16/30 Write block 556 len 28672 data: 3 5 5 5 4 5 5
69222024-01-24T19:08:26.661Z17/30 Write block 393 len 16384 data: 7 7 6 7
69232024-01-24T19:08:26.661Z18/30 Write block 544 len 16384 data: 7 6 5 7
69242024-01-24T19:08:26.662Z19/30 Write block 319 len 20480 data: 8 8 8 7 8
69252024-01-24T19:08:26.662Z20/30 Write block 475 len 32768 data: 8 7 7 7 6 7 8 8
69262024-01-24T19:08:26.662Z21/30 Write block 48 len 12288 data: 5 5 7
69272024-01-24T19:08:26.662Z22/30 Read block 268 len 28672
69282024-01-24T19:08:26.662Z23/30 Read block 200 len 32768
69292024-01-24T19:08:26.662Z24/30 Read block 569 len 32768
69302024-01-24T19:08:26.663Z25/30 Flush
69312024-01-24T19:08:26.663Z26/30 Write block 249 len 4096 data: 8
69322024-01-24T19:08:26.663Z27/30 Write block 572 len 28672 data: 4 3 4 5 5 3 4
69332024-01-24T19:08:26.663Z28/30 Write block 531 len 40960 data: 4 4 4 4 4 5 6 5 5 3
69342024-01-24T19:08:26.663Z29/30 Read block 430 len 8192
69352024-01-24T19:08:26.663Z30/30 Read block 493 len 16384
69362024-01-24T19:08:26.663Z----------------------------------------------------------------
69372024-01-24T19:08:26.664Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69382024-01-24T19:08:26.664ZStates: Active Active Active
69392024-01-24T19:08:26.664ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69402024-01-24T19:08:26.664Z 1 Acked 1000 Flush 0 Done Sent Done false
69412024-01-24T19:08:26.664Z 2 Acked 1001 Write 6 Done Sent Done false
69422024-01-24T19:08:26.664Z 3 Acked 1002 Read 9 Done Sent Done false
69432024-01-24T19:08:26.665Z 4 Acked 1003 Write 2 Done Sent Done false
69442024-01-24T19:08:26.665Z 5 Acked 1004 Read 10 Done Sent Done false
69452024-01-24T19:08:26.665Z 6 Acked 1005 Read 4 Done Sent Done false
69462024-01-24T19:08:26.665Z 7 Acked 1006 Flush 0 Done Sent Done false
69472024-01-24T19:08:26.665Z 8 Acked 1007 Write 3 Done Sent Done false
69482024-01-24T19:08:26.665Z 9 Acked 1008 Write 6 Done Sent Done false
69492024-01-24T19:08:26.666Z 10 Acked 1009 Write 7 Done Sent Done false
69502024-01-24T19:08:26.667Z 11 Acked 1010 Read 1 Done Sent Done false
69512024-01-24T19:08:26.667Z 12 Acked 1011 Flush 0 Done Sent Done false
69522024-01-24T19:08:26.667Z 13 Acked 1012 Write 2 Done Sent Done false
69532024-01-24T19:08:26.667Z 14 Acked 1013 Flush 0 Done Sent Done false
69542024-01-24T19:08:26.667Z 15 Acked 1014 Flush 0 Done Sent Done false
69552024-01-24T19:08:26.667Z 16 Acked 1015 Write 7 Done Sent Done false
69562024-01-24T19:08:26.668Z 17 Acked 1016 Write 4 Done Sent Done false
69572024-01-24T19:08:26.668Z 18 Acked 1017 Write 4 Done Sent Done false
69582024-01-24T19:08:26.668Z 19 Acked 1018 Write 5 Done Sent Done false
69592024-01-24T19:08:26.668Z 20 Acked 1019 Write 8 Done Sent Done false
69602024-01-24T19:08:26.668Z 21 Acked 1020 Write 3 Done Sent Done false
69612024-01-24T19:08:26.668Z 22 Acked 1021 Read 7 Done Sent Done false
69622024-01-24T19:08:26.668Z 23 Acked 1022 Read 8 Done Sent Done false
69632024-01-24T19:08:26.669Z 24 Acked 1023 Read 8 Done Sent Done false
69642024-01-24T19:08:26.669Z 25 Acked 1024 Flush 0 Done Sent Done false
69652024-01-24T19:08:26.669Z 26 Acked 1025 Write 1 Done Sent Done false
69662024-01-24T19:08:26.669Z 27 Acked 1026 Write 7 Done Sent Done false
69672024-01-24T19:08:26.669Z 28 Acked 1027 Write 10 Done Sent Done false
69682024-01-24T19:08:26.669Z 29 Acked 1028 Read 2 Done Sent Done false
69692024-01-24T19:08:26.669Z 30 Acked 1029 Read 4 Done Sent Done false
69702024-01-24T19:08:26.670Z STATES DS:0 DS:1 DS:2 TOTAL
69712024-01-24T19:08:26.670Z New 0 0 0 0
69722024-01-24T19:08:26.670Z Sent 0 30 0 30
69732024-01-24T19:08:26.671Z Done 30 0 30 60
69742024-01-24T19:08:26.671Z Skipped 0 0 0 0
69752024-01-24T19:08:26.671Z Error 0 0 0 0
69762024-01-24T19:08:26.672ZLast Flush: 1024 0 1024
69772024-01-24T19:08:26.672ZDownstairs last five completed:
69782024-01-24T19:08:26.672ZUpstairs last five completed: 30 29 28 27 26
69792024-01-24T19:08:26.672ZWrote out file "/tmp/test_repair_verify.data"
69802024-01-24T19:08:26.672Z
69812024-01-24T19:08:26.681ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
69822024-01-24T19:08:26.689ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69832024-01-24T19:08:26.689Z 0 000-019 62 62 62 102 102 102 F F F
69842024-01-24T19:08:26.689Z 1 020-039 60 60 60 94 94 94 F F F
69852024-01-24T19:08:26.689Z 2 040-059 64 55 64 110 88 110 F F F <---
69862024-01-24T19:08:26.690Z 3 060-079 64 62 64 107 103 107 F F F <---
69872024-01-24T19:08:26.690Z 4 080-099 60 60 60 95 95 95 F F F
69882024-01-24T19:08:26.690Z 5 100-119 63 63 63 104 104 104 F F F
69892024-01-24T19:08:26.690Z 6 120-139 63 63 63 104 104 104 F F F
69902024-01-24T19:08:26.690Z 7 140-159 60 60 60 98 98 98 F F F
69912024-01-24T19:08:26.691Z 8 160-179 62 62 62 103 103 103 F F F
69922024-01-24T19:08:26.691Z 9 180-199 56 56 56 89 89 89 F F F
69932024-01-24T19:08:26.691Z 10 200-219 62 62 62 103 103 103 F F F
69942024-01-24T19:08:26.691Z 11 220-239 56 56 56 89 89 89 F F F
69952024-01-24T19:08:26.691Z 12 240-259 62 62 62 103 103 103 T F T <---
69962024-01-24T19:08:26.691Z 13 260-279 62 62 62 103 103 103 F F F
69972024-01-24T19:08:26.692Z 14 280-299 53 53 53 86 86 86 F F F
69982024-01-24T19:08:26.692Z 15 300-319 64 61 64 110 101 110 F F F <---
69992024-01-24T19:08:26.692Z 16 320-339 64 58 64 110 91 110 F F F <---
70002024-01-24T19:08:26.692Z 17 340-359 55 55 55 88 88 88 F F F
70012024-01-24T19:08:26.692Z 18 360-379 64 60 64 107 98 107 F F F <---
70022024-01-24T19:08:26.692Z 19 380-399 64 60 64 110 93 110 F F F <---
70032024-01-24T19:08:26.693Z 20 400-419 64 60 64 107 99 107 F F F <---
70042024-01-24T19:08:26.693Z 21 420-439 62 62 62 103 103 103 F F F
70052024-01-24T19:08:26.693Z 22 440-459 59 59 59 92 92 92 F F F
70062024-01-24T19:08:26.693Z 23 460-479 64 60 64 110 98 110 F F F <---
70072024-01-24T19:08:26.693Z 24 480-499 64 62 64 110 103 110 F F F <---
70082024-01-24T19:08:26.693Z 25 500-519 62 62 62 103 103 103 F F F
70092024-01-24T19:08:26.693Z 26 520-539 64 62 64 106 103 106 T F T <---
70102024-01-24T19:08:26.694Z 27 540-559 64 61 64 110 101 110 T F T <---
70112024-01-24T19:08:26.694Z 28 560-579 64 44 64 110 72 110 T F T <---
70122024-01-24T19:08:26.694Z 29 580-599 59 59 59 92 92 92 F F F
70132024-01-24T19:08:26.694ZMax gen: 64, Max flush: 110
70142024-01-24T19:08:26.694ZError: Difference in extent metadata found!
70152024-01-24T19:08:26.694ZA Difference in extent metadata is expected here
70162024-01-24T19:08:26.695ZOn loop 32
70172024-01-24T19:08:26.695Z
70182024-01-24T19:08:26.705ZVerifying data now
70192024-01-24T19:08:31.832ZNo change after verify
70202024-01-24T19:08:31.833ZLoop: 32 Downstairs dump after verify (and repair):
70212024-01-24T19:08:31.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70222024-01-24T19:08:31.856Z 0 000-019 62 62 62 102 102 102 F F F
70232024-01-24T19:08:32.405Z 1 020-039 60 60 60 94 94 94 F F F
70242024-01-24T19:08:32.405Z 2 040-059 64 64 64 110 110 110 F F F
70252024-01-24T19:08:32.405Z 3 060-079 64 64 64 107 107 107 F F F
70262024-01-24T19:08:32.405Z 4 080-099 60 60 60 95 95 95 F F F
70272024-01-24T19:08:32.405Z 5 100-119 63 63 63 104 104 104 F F F
70282024-01-24T19:08:32.405Z 6 120-139 63 63 63 104 104 104 F F F
70292024-01-24T19:08:32.406Z 7 140-159 60 60 60 98 98 98 F F F
70302024-01-24T19:08:32.406Z 8 160-179 62 62 62 103 103 103 F F F
70312024-01-24T19:08:32.406Z 9 180-199 56 56 56 89 89 89 F F F
70322024-01-24T19:08:32.406Z 10 200-219 62 62 62 103 103 103 F F F
70332024-01-24T19:08:32.406Z 11 220-239 56 56 56 89 89 89 F F F
70342024-01-24T19:08:32.407Z 12 240-259 65 65 65 111 111 111 F F F
70352024-01-24T19:08:32.407Z 13 260-279 62 62 62 103 103 103 F F F
70362024-01-24T19:08:32.407Z 14 280-299 53 53 53 86 86 86 F F F
70372024-01-24T19:08:32.407Z 15 300-319 64 64 64 110 110 110 F F F
70382024-01-24T19:08:32.407Z 16 320-339 64 64 64 110 110 110 F F F
70392024-01-24T19:08:32.408Z 17 340-359 55 55 55 88 88 88 F F F
70402024-01-24T19:08:32.408Z 18 360-379 64 64 64 107 107 107 F F F
70412024-01-24T19:08:32.408Z 19 380-399 64 64 64 110 110 110 F F F
70422024-01-24T19:08:32.408Z 20 400-419 64 64 64 107 107 107 F F F
70432024-01-24T19:08:32.408Z 21 420-439 62 62 62 103 103 103 F F F
70442024-01-24T19:08:32.408Z 22 440-459 59 59 59 92 92 92 F F F
70452024-01-24T19:08:32.409Z 23 460-479 64 64 64 110 110 110 F F F
70462024-01-24T19:08:32.409Z 24 480-499 64 64 64 110 110 110 F F F
70472024-01-24T19:08:32.409Z 25 500-519 62 62 62 103 103 103 F F F
70482024-01-24T19:08:32.409Z 26 520-539 65 65 65 111 111 111 F F F
70492024-01-24T19:08:32.409Z 27 540-559 65 65 65 111 111 111 F F F
70502024-01-24T19:08:32.409Z 28 560-579 65 65 65 111 111 111 F F F
70512024-01-24T19:08:32.409Z 29 580-599 59 59 59 92 92 92 F F F
70522024-01-24T19:08:32.409ZMax gen: 65, Max flush: 111
70532024-01-24T19:08:32.409Z
70542024-01-24T19:08:32.409ZBegin loop 33
70552024-01-24T19:08:32.409ZDownstairs to restart: 1
70562024-01-24T19:08:33.256Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.550167479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393}
70572024-01-24T19:08:33.256Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.5506169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393}
70582024-01-24T19:08:33.256Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.550684523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393}
70592024-01-24T19:08:33.257Z{"msg":"Crucible 4150b27e-56fd-45e4-b257-a72d7de9f745 has session id: b80f28b7-19db-464d-a464-5d05b36af67e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.550711876Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70602024-01-24T19:08:33.257Z{"msg":"Upstairs opts: Upstairs UUID: 4150b27e-56fd-45e4-b257-a72d7de9f745, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.550787827Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70612024-01-24T19:08:33.257Z{"msg":"Crucible stats registered with UUID: 4150b27e-56fd-45e4-b257-a72d7de9f745","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.55082267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70622024-01-24T19:08:33.257ZCrucible runtime is spawned
70632024-01-24T19:08:33.257Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551102748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393}
70642024-01-24T19:08:33.257Z{"msg":"4150b27e-56fd-45e4-b257-a72d7de9f745 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551407355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70652024-01-24T19:08:33.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551485243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70662024-01-24T19:08:33.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551602753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70672024-01-24T19:08:33.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551683459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70682024-01-24T19:08:33.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551780893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70692024-01-24T19:08:33.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.551868538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70702024-01-24T19:08:33.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552165971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70712024-01-24T19:08:33.257Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.55224528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70722024-01-24T19:08:33.257Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552334467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70732024-01-24T19:08:33.257Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552418909Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70742024-01-24T19:08:33.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552508452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70752024-01-24T19:08:33.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552555588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70762024-01-24T19:08:33.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.552626573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70772024-01-24T19:08:33.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553354297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70782024-01-24T19:08:33.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553420448Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70792024-01-24T19:08:33.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553455152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70802024-01-24T19:08:33.257Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553525608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70812024-01-24T19:08:33.258Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553551289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70822024-01-24T19:08:33.258Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.553582518Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70832024-01-24T19:08:33.258Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.554565531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70842024-01-24T19:08:33.258Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.55481116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70852024-01-24T19:08:33.258Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.554891472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70862024-01-24T19:08:33.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555345155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70872024-01-24T19:08:33.258Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555411892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70882024-01-24T19:08:33.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555457042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70892024-01-24T19:08:33.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555765754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70902024-01-24T19:08:33.258Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555881684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70912024-01-24T19:08:33.258Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.555926634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70922024-01-24T19:08:33.258Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556056141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70932024-01-24T19:08:33.258Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556148649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70942024-01-24T19:08:33.258Z{"msg":"[0]R flush_numbers[0..12]: [102, 94, 110, 107, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556213571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70952024-01-24T19:08:33.258Z{"msg":"[0]R generation[0..12]: [62, 60, 64, 64, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556242323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70962024-01-24T19:08:33.258Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556288034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70972024-01-24T19:08:33.260Z{"msg":"[1]R flush_numbers[0..12]: [102, 94, 110, 107, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556314818Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
70982024-01-24T19:08:33.260Z{"msg":"[1]R generation[0..12]: [62, 60, 64, 64, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
70992024-01-24T19:08:33.260Z,"time":"2024-01-24T19:08:30.556358938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71002024-01-24T19:08:33.260Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556388055Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71012024-01-24T19:08:33.260Z{"msg":"[2]R flush_numbers[0..12]: [102, 94, 110, 107, 95, 104, 104, 98, 103, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556431839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71022024-01-24T19:08:33.260Z{"msg":"[2]R generation[0..12]: [62, 60, 64, 64, 60, 63, 63, 60, 62, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556460628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71032024-01-24T19:08:33.260Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556505656Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71042024-01-24T19:08:33.261Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556530742Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71052024-01-24T19:08:33.261Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556553176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71062024-01-24T19:08:33.261Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556596568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71072024-01-24T19:08:33.262Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556621327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71082024-01-24T19:08:33.262Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556672672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71092024-01-24T19:08:33.262Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556696308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71102024-01-24T19:08:33.262Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.55671803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71112024-01-24T19:08:33.262Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556760673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71122024-01-24T19:08:33.262Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556785528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71132024-01-24T19:08:33.262Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556836173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71142024-01-24T19:08:33.262Z{"msg":"4150b27e-56fd-45e4-b257-a72d7de9f745 is now active with session: b80f28b7-19db-464d-a464-5d05b36af67e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556860923Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71152024-01-24T19:08:33.262Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556880674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393,"session_id":"b80f28b7-19db-464d-a464-5d05b36af67e"}
71162024-01-24T19:08:33.262Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:30.556922071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1393}
71172024-01-24T19:08:33.262ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71182024-01-24T19:08:33.262ZLoading write count information from file "/tmp/test_repair_verify.data"
71192024-01-24T19:08:33.262ZRun Repair workload
71202024-01-24T19:08:33.262Z01/30 Write block 67 len 16384 data: 2 2 2 4
71212024-01-24T19:08:33.262Z02/30 Write block 486 len 4096 data: 5
71222024-01-24T19:08:33.262Z03/30 Write block 481 len 8192 data: 9 9
71232024-01-24T19:08:33.262Z04/30 Read block 223 len 24576
71242024-01-24T19:08:33.262Z05/30 Read block 534 len 16384
71252024-01-24T19:08:33.263Z06/30 Read block 380 len 20480
71262024-01-24T19:08:33.263Z07/30 Read block 593 len 28672
71272024-01-24T19:08:33.263Z08/30 Write block 446 len 36864 data: 5 6 7 7 6 6 7 8 7
71282024-01-24T19:08:33.263Z09/30 Read block 95 len 32768
71292024-01-24T19:08:33.263Z10/30 Write block 377 len 4096 data: 7
71302024-01-24T19:08:33.263Z11/30 Write block 471 len 32768 data: 6 10 7 8 9 8 8 8
71312024-01-24T19:08:33.263Z12/30 Read block 231 len 12288
71322024-01-24T19:08:33.263Z13/30 Read block 536 len 24576
71332024-01-24T19:08:33.263Z14/30 Read block 275 len 4096
71342024-01-24T19:08:33.263Z15/30 Read block 339 len 8192
71352024-01-24T19:08:33.263Z16/30 Read block 121 len 4096
71362024-01-24T19:08:33.263Z17/30 Write block 467 len 40960 data: 6 6 6 5 7 11 8 9 10 9
71372024-01-24T19:08:33.263Z18/30 Write block 364 len 40960 data: 8 7 7 7 7 7 8 8 7 8
71382024-01-24T19:08:33.263Z19/30 Write block 538 len 36864 data: 6 6 4 6 8 8 8 7 6
71392024-01-24T19:08:33.263Z20/30 Read block 216 len 4096
71402024-01-24T19:08:33.263Z21/30 Write block 343 len 40960 data: 6 6 6 3 2 2 3 3 3 2
71412024-01-24T19:08:33.263Z22/30 Write block 313 len 24576 data: 5 5 7 8 8 8
71422024-01-24T19:08:33.263Z23/30 Read block 577 len 32768
71432024-01-24T19:08:33.264Z24/30 Write block 157 len 36864 data: 5 4 3 3 7 8 8 7 7
71442024-01-24T19:08:33.264Z25/30 Read block 434 len 36864
71452024-01-24T19:08:33.264Z26/30 Write block 314 len 4096 data: 6
71462024-01-24T19:08:33.265Z27/30 Read block 399 len 28672
71472024-01-24T19:08:33.265Z28/30 Read block 543 len 20480
71482024-01-24T19:08:33.265Z29/30 Read block 575 len 36864
71492024-01-24T19:08:33.265Z30/30 Write block 44 len 12288 data: 7 8 6
71502024-01-24T19:08:33.265Z----------------------------------------------------------------
71512024-01-24T19:08:33.265Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71522024-01-24T19:08:33.265ZStates: Active Active Active
71532024-01-24T19:08:33.265ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71542024-01-24T19:08:33.265Z 1 Acked 1000 Write 4 Done Done Done false
71552024-01-24T19:08:33.265Z 2 Acked 1001 Write 1 Done Done Done false
71562024-01-24T19:08:33.271Z 3 Acked 1002 Write 2 Done Sent Done false
71572024-01-24T19:08:33.271Z 4 Acked 1003 Read 6 Done Sent Done false
71582024-01-24T19:08:33.271Z 5 Acked 1004 Read 4 Done Sent Done false
71592024-01-24T19:08:33.271Z 6 Acked 1005 Read 5 Done Sent Done false
71602024-01-24T19:08:33.271Z 7 Acked 1006 Read 7 Done Sent Done false
71612024-01-24T19:08:33.271Z 8 Acked 1007 Write 9 Done Sent Done false
71622024-01-24T19:08:33.271Z 9 Acked 1008 Read 8 Done Sent Done false
71632024-01-24T19:08:33.271Z 10 Acked 1009 Write 1 Done Sent Done false
71642024-01-24T19:08:33.271Z 11 Acked 1010 Write 8 Done Sent Done false
71652024-01-24T19:08:33.271Z 12 Acked 1011 Read 3 Done Sent Done false
71662024-01-24T19:08:33.271Z 13 Acked 1012 Read 6 Done Sent Done false
71672024-01-24T19:08:33.271Z 14 Acked 1013 Read 1 Done Sent Done false
71682024-01-24T19:08:33.271Z 15 Acked 1014 Read 2 Done Sent Done false
71692024-01-24T19:08:33.271Z 16 Acked 1015 Read 1 Done Sent Done false
71702024-01-24T19:08:33.271Z 17 Acked 1016 Write 10 Done Sent Done false
71712024-01-24T19:08:33.271Z 18 Acked 1017 Write 10 Done Sent Done false
71722024-01-24T19:08:33.271Z 19 Acked 1018 Write 9 Done Sent Done false
71732024-01-24T19:08:33.271Z 20 Acked 1019 Read 1 Done Sent Done false
71742024-01-24T19:08:33.272Z 21 Acked 1020 Write 10 Done Sent Done false
71752024-01-24T19:08:33.272Z 22 Acked 1021 Write 6 Done Sent Done false
71762024-01-24T19:08:33.272Z 23 Acked 1022 Read 8 Done Sent Done false
71772024-01-24T19:08:33.272Z 24 Acked 1023 Write 9 Done Sent Done false
71782024-01-24T19:08:33.272Z 25 Acked 1024 Read 9 Done Sent Done false
71792024-01-24T19:08:33.272Z 26 Acked 1025 Write 1 Done Sent Done false
71802024-01-24T19:08:33.272Z 27 Acked 1026 Read 7 Done Sent Done false
71812024-01-24T19:08:33.272Z 28 Acked 1027 Read 5 Done Sent Done false
71822024-01-24T19:08:33.272Z 29 Acked 1028 Read 9 Done Sent Done false
71832024-01-24T19:08:33.272Z 30 Acked 1029 Write 3 Sent Sent Sent false
71842024-01-24T19:08:33.272Z STATES DS:0 DS:1 DS:2 TOTAL
71852024-01-24T19:08:33.272Z New 0 0 0 0
71862024-01-24T19:08:33.272Z Sent 1 28 1 30
71872024-01-24T19:08:33.272Z Done 29 2 29 60
71882024-01-24T19:08:33.272Z Skipped 0 0 0 0
71892024-01-24T19:08:33.272Z Error 0 0 0 0
71902024-01-24T19:08:33.272ZLast Flush: 0 0 0
71912024-01-24T19:08:33.272ZDownstairs last five completed:
71922024-01-24T19:08:33.272ZUpstairs last five completed: 30 29 28 27 26
71932024-01-24T19:08:33.272ZWrote out file "/tmp/test_repair_verify.data"
71942024-01-24T19:08:33.272Z
71952024-01-24T19:08:33.272ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
71962024-01-24T19:08:33.272ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71972024-01-24T19:08:33.273Z 0 000-019 62 62 62 102 102 102 F F F
71982024-01-24T19:08:33.273Z 1 020-039 60 60 60 94 94 94 F F F
71992024-01-24T19:08:33.273Z 2 040-059 64 64 64 110 110 110 T F T <---
72002024-01-24T19:08:33.273Z 3 060-079 64 64 64 107 107 107 T T T
72012024-01-24T19:08:33.273Z 4 080-099 60 60 60 95 95 95 F F F
72022024-01-24T19:08:33.273Z 5 100-119 63 63 63 104 104 104 F F F
72032024-01-24T19:08:33.273Z 6 120-139 63 63 63 104 104 104 F F F
72042024-01-24T19:08:33.273Z 7 140-159 60 60 60 98 98 98 T F T <---
72052024-01-24T19:08:33.273Z 8 160-179 62 62 62 103 103 103 T F T <---
72062024-01-24T19:08:33.273Z 9 180-199 56 56 56 89 89 89 F F F
72072024-01-24T19:08:33.273Z 10 200-219 62 62 62 103 103 103 F F F
72082024-01-24T19:08:33.273Z 11 220-239 56 56 56 89 89 89 F F F
72092024-01-24T19:08:33.273Z 12 240-259 65 65 65 111 111 111 F F F
72102024-01-24T19:08:33.273Z 13 260-279 62 62 62 103 103 103 F F F
72112024-01-24T19:08:33.273Z 14 280-299 53 53 53 86 86 86 F F F
72122024-01-24T19:08:33.273Z 15 300-319 64 64 64 110 110 110 T F T <---
72132024-01-24T19:08:33.274Z 16 320-339 64 64 64 110 110 110 F F F
72142024-01-24T19:08:33.274Z 17 340-359 55 55 55 88 88 88 T F T <---
72152024-01-24T19:08:33.274Z 18 360-379 64 64 64 107 107 107 T F T <---
72162024-01-24T19:08:33.274Z 19 380-399 64 64 64 110 110 110 F F F
72172024-01-24T19:08:33.274Z 20 400-419 64 64 64 107 107 107 F F F
72182024-01-24T19:08:33.274Z 21 420-439 62 62 62 103 103 103 F F F
72192024-01-24T19:08:33.274Z 22 440-459 59 59 59 92 92 92 T F T <---
72202024-01-24T19:08:33.274Z 23 460-479 64 64 64 110 110 110 T F T <---
72212024-01-24T19:08:33.274Z 24 480-499 64 64 64 110 110 110 T T T
72222024-01-24T19:08:33.274Z 25 500-519 62 62 62 103 103 103 F F F
72232024-01-24T19:08:33.275Z 26 520-539 65 65 65 111 111 111 T F T <---
72242024-01-24T19:08:33.275Z 27 540-559 65 65 65 111 111 111 T F T <---
72252024-01-24T19:08:33.275Z 28 560-579 65 65 65 111 111 111 F F F
72262024-01-24T19:08:33.275Z 29 580-599 59 59 59 92 92 92 F F F
72272024-01-24T19:08:33.275ZMax gen: 65, Max flush: 111
72282024-01-24T19:08:33.275ZError: Difference in extent metadata found!
72292024-01-24T19:08:33.275ZA Difference in extent metadata is expected here
72302024-01-24T19:08:33.275ZOn loop 33
72312024-01-24T19:08:33.275Z
72322024-01-24T19:08:33.275ZVerifying data now
72332024-01-24T19:08:37.086ZNo change after verify
72342024-01-24T19:08:37.086ZLoop: 33 Downstairs dump after verify (and repair):
72352024-01-24T19:08:37.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72362024-01-24T19:08:37.113Z 0 000-019 62 62 62 102 102 102 F F F
72372024-01-24T19:08:37.719Z 1 020-039 60 60 60 94 94 94 F F F
72382024-01-24T19:08:37.719Z 2 040-059 66 66 66 112 112 112 F F F
72392024-01-24T19:08:37.719Z 3 060-079 66 66 66 112 112 112 F F F
72402024-01-24T19:08:37.719Z 4 080-099 60 60 60 95 95 95 F F F
72412024-01-24T19:08:37.720Z 5 100-119 63 63 63 104 104 104 F F F
72422024-01-24T19:08:37.720Z 6 120-139 63 63 63 104 104 104 F F F
72432024-01-24T19:08:37.721Z 7 140-159 66 66 66 112 112 112 F F F
72442024-01-24T19:08:37.721Z 8 160-179 66 66 66 112 112 112 F F F
72452024-01-24T19:08:37.721Z 9 180-199 56 56 56 89 89 89 F F F
72462024-01-24T19:08:37.721Z 10 200-219 62 62 62 103 103 103 F F F
72472024-01-24T19:08:37.721Z 11 220-239 56 56 56 89 89 89 F F F
72482024-01-24T19:08:37.722Z 12 240-259 65 65 65 111 111 111 F F F
72492024-01-24T19:08:37.722Z 13 260-279 62 62 62 103 103 103 F F F
72502024-01-24T19:08:37.722Z 14 280-299 53 53 53 86 86 86 F F F
72512024-01-24T19:08:37.722Z 15 300-319 66 66 66 112 112 112 F F F
72522024-01-24T19:08:37.722Z 16 320-339 64 64 64 110 110 110 F F F
72532024-01-24T19:08:37.722Z 17 340-359 66 66 66 112 112 112 F F F
72542024-01-24T19:08:37.723Z 18 360-379 66 66 66 112 112 112 F F F
72552024-01-24T19:08:37.723Z 19 380-399 64 64 64 110 110 110 F F F
72562024-01-24T19:08:37.723Z 20 400-419 64 64 64 107 107 107 F F F
72572024-01-24T19:08:37.723Z 21 420-439 62 62 62 103 103 103 F F F
72582024-01-24T19:08:37.723Z 22 440-459 66 66 66 112 112 112 F F F
72592024-01-24T19:08:37.726Z 23 460-479 66 66 66 112 112 112 F F F
72602024-01-24T19:08:37.726Z 24 480-499 66 66 66 112 112 112 F F F
72612024-01-24T19:08:37.726Z 25 500-519 62 62 62 103 103 103 F F F
72622024-01-24T19:08:37.726Z 26 520-539 66 66 66 112 112 112 F F F
72632024-01-24T19:08:37.726Z 27 540-559 66 66 66 112 112 112 F F F
72642024-01-24T19:08:37.726Z 28 560-579 65 65 65 111 111 111 F F F
72652024-01-24T19:08:37.727Z 29 580-599 59 59 59 92 92 92 F F F
72662024-01-24T19:08:37.727ZMax gen: 66, Max flush: 112
72672024-01-24T19:08:37.727Z
72682024-01-24T19:08:37.728ZBegin loop 34
72692024-01-24T19:08:37.728ZDownstairs to restart: 2
72702024-01-24T19:08:37.751Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.806308997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401}
72712024-01-24T19:08:37.751Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.806839479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401}
72722024-01-24T19:08:37.751Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.806913093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401}
72732024-01-24T19:08:37.751Z{"msg":"Crucible c9663d50-ec57-432c-821d-e60cf1062cc7 has session id: 26689db9-7add-44d2-8264-f6180009440d","v":0,"name":"crucible","level":30Crucible runtime is spawned
72742024-01-24T19:08:37.752Z,"time":"2024-01-24T19:08:35.807006235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72752024-01-24T19:08:37.752Z{"msg":"Upstairs opts: Upstairs UUID: c9663d50-ec57-432c-821d-e60cf1062cc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807143314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72762024-01-24T19:08:37.752Z{"msg":"Crucible stats registered with UUID: c9663d50-ec57-432c-821d-e60cf1062cc7","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807174402Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72772024-01-24T19:08:37.752Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807196071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401}
72782024-01-24T19:08:37.752Z{"msg":"c9663d50-ec57-432c-821d-e60cf1062cc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807400089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72792024-01-24T19:08:37.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807457127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72802024-01-24T19:08:37.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807490848Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72812024-01-24T19:08:37.754Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807617761Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72822024-01-24T19:08:37.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807663323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72832024-01-24T19:08:37.754Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807764076Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72842024-01-24T19:08:37.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807855294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72852024-01-24T19:08:37.754Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.807974007Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72862024-01-24T19:08:37.754Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.808019726Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72872024-01-24T19:08:37.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.808092206Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72882024-01-24T19:08:37.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.808328023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72892024-01-24T19:08:37.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.80843664Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72902024-01-24T19:08:37.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.808599234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72912024-01-24T19:08:37.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809324591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72922024-01-24T19:08:37.755Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809499523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72932024-01-24T19:08:37.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809543837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72942024-01-24T19:08:37.756Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809582425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72952024-01-24T19:08:37.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809652811Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72962024-01-24T19:08:37.756Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.809698935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72972024-01-24T19:08:37.756Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.810685829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72982024-01-24T19:08:37.756Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.810764557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
72992024-01-24T19:08:37.756Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.810939662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73002024-01-24T19:08:37.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.811630466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73012024-01-24T19:08:37.757Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.811710207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73022024-01-24T19:08:37.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.811739874Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73032024-01-24T19:08:37.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.81194166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73042024-01-24T19:08:37.757Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.81200894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73052024-01-24T19:08:37.757Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812038036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73062024-01-24T19:08:37.757Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.81208149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73072024-01-24T19:08:37.758Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812108462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73082024-01-24T19:08:37.758Z{"msg":"[0]R flush_numbers[0..12]: [102, 94, 112, 112, 95, 104, 104, 112, 112, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812153232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73092024-01-24T19:08:37.758Z{"msg":"[0]R generation[0..12]: [62, 60, 66, 66, 60, 63, 63, 66, 66, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812180443Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73102024-01-24T19:08:37.759Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812225475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73112024-01-24T19:08:37.759Z{"msg":"[1]R flush_numbers[0..12]: [102, 94, 112, 112, 95, 104, 104, 112, 112, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.81225434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73122024-01-24T19:08:37.759ZWait for a query_work_queue command to finish before sending IO
73132024-01-24T19:08:37.759Z{"msg":"[1]R generation[0..12]: [62, 60, 66, 66, 60, 63, 63, 66, 66, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812331538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73142024-01-24T19:08:37.759Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812406858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73152024-01-24T19:08:37.759Z{"msg":"[2]R flush_numbers[0..12]: [102, 94, 112, 112, 95, 104, 104, 112, 112, 89, 103, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.81243707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73162024-01-24T19:08:37.759Z{"msg":"[2]R generation[0..12]: [62, 60, 66, 66, 60, 63, 63, 66, 66, 56, 62, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812476733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73172024-01-24T19:08:37.759Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812555225Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73182024-01-24T19:08:37.760Z{"msg":"Max found gen is 67","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812621804Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73192024-01-24T19:08:37.760Z{"msg":"Generation requested: 68 >= found:67","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812661451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73202024-01-24T19:08:37.761Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812733597Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73212024-01-24T19:08:37.762Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812811686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73222024-01-24T19:08:37.762Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812888239Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73232024-01-24T19:08:37.762Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812931911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73242024-01-24T19:08:37.762Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.812968999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73252024-01-24T19:08:37.762Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.813036091Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73262024-01-24T19:08:37.762Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.813069971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73272024-01-24T19:08:37.762Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73282024-01-24T19:08:37.762Z,"level":30,"time":"2024-01-24T19:08:35.813220426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73292024-01-24T19:08:37.762Z{"msg":"c9663d50-ec57-432c-821d-e60cf1062cc7 is now active with session: 26689db9-7add-44d2-8264-f6180009440d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.813300737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73302024-01-24T19:08:37.763Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.813373702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401,"session_id":"26689db9-7add-44d2-8264-f6180009440d"}
73312024-01-24T19:08:37.763Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:35.813444952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1401}
73322024-01-24T19:08:37.764ZLoading write count information from file "/tmp/test_repair_verify.data"
73332024-01-24T19:08:37.764ZRun Repair workload
73342024-01-24T19:08:37.764Z01/30 Write block 48 len 40960 data: 6 6 8 6 5 5 5 6 8 8
73352024-01-24T19:08:37.764Z02/30 Read block 244 len 36864
73362024-01-24T19:08:37.764Z03/30 Read block 463 len 28672
73372024-01-24T19:08:37.764Z04/30 Write block 263 len 40960 data: 3 6 6 7 6 6 6 6 6 6
73382024-01-24T19:08:37.764Z05/30 Read block 524 len 24576
73392024-01-24T19:08:37.765Z06/30 Read block 491 len 40960
73402024-01-24T19:08:37.765Z07/30 Read block 196 len 28672
73412024-01-24T19:08:37.765Z08/30 Write block 240 len 20480 data: 4 5 6 8 8
73422024-01-24T19:08:37.765Z09/30 Read block 135 len 36864
73432024-01-24T19:08:37.765Z10/30 Read block 239 len 16384
73442024-01-24T19:08:37.765Z11/30 Write block 434 len 32768 data: 8 6 5 6 4 4 3 3
73452024-01-24T19:08:37.766Z12/30 Read block 481 len 24576
73462024-01-24T19:08:37.766Z13/30 Write block 204 len 36864 data: 5 5 4 4 4 5 7 6 6
73472024-01-24T19:08:37.766Z14/30 Read block 555 len 20480
73482024-01-24T19:08:37.766Z15/30 Flush
73492024-01-24T19:08:37.766Z16/30 Write block 171 len 32768 data: 8 7 6 8 7 7 5 5
73502024-01-24T19:08:37.766Z17/30 Flush
73512024-01-24T19:08:37.766Z18/30 Write block 82 len 12288 data: 9 10 9
73522024-01-24T19:08:37.767Z19/30 Read block 578 len 12288
73532024-01-24T19:08:37.767Z20/30 Read block 63 len 40960
73542024-01-24T19:08:37.767Z21/30 Read block 571 len 20480
73552024-01-24T19:08:37.768Z22/30 Read block 327 len 8192
73562024-01-24T19:08:37.768Z23/30 Write block 594 len 12288 data: 7 6 5
73572024-01-24T19:08:37.768Z24/30 Write block 515 len 24576 data: 8 6 8 9 9 8
73582024-01-24T19:08:37.768Z25/30 Write block 92 len 20480 data: 6 7 8 7 7
73592024-01-24T19:08:37.768Z26/30 Read block 211 len 16384
73602024-01-24T19:08:37.769Z27/30 Read block 353 len 4096
73612024-01-24T19:08:37.769Z28/30 Read block 574 len 24576
73622024-01-24T19:08:37.769Z29/30 Read block 188 len 32768
73632024-01-24T19:08:37.769Z30/30 Write block 461 len 8192 data: 6 5
73642024-01-24T19:08:37.769Z----------------------------------------------------------------
73652024-01-24T19:08:37.769Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
73662024-01-24T19:08:37.769ZStates: Active Active Active
73672024-01-24T19:08:37.770ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
73682024-01-24T19:08:37.770Z 1 Acked 1000 Write 10 Done Done Sent false
73692024-01-24T19:08:37.770Z 2 Acked 1001 Read 9 Done Done Sent false
73702024-01-24T19:08:37.779Z 3 Acked 1002 Read 7 Done Done Sent false
73712024-01-24T19:08:37.780Z 4 Acked 1003 Write 10 Done Done Sent false
73722024-01-24T19:08:37.780Z 5 Acked 1004 Read 6 Done Done Sent false
73732024-01-24T19:08:37.780Z 6 Acked 1005 Read 10 Done Done Sent false
73742024-01-24T19:08:37.780Z 7 Acked 1006 Read 7 Done Done Sent false
73752024-01-24T19:08:37.780Z 8 Acked 1007 Write 5 Done Done Sent false
73762024-01-24T19:08:37.780Z 9 Acked 1008 Read 9 Done Done Sent false
73772024-01-24T19:08:37.781Z 10 Acked 1009 Read 4 Done Done Sent false
73782024-01-24T19:08:37.781Z 11 Acked 1010 Write 8 Done Done Sent false
73792024-01-24T19:08:37.781Z 12 Acked 1011 Read 6 Done Done Sent false
73802024-01-24T19:08:37.781Z 13 Acked 1012 Write 9 Done Done Sent false
73812024-01-24T19:08:37.781Z 14 Acked 1013 Read 5 Done Done Sent false
73822024-01-24T19:08:37.782Z 15 Acked 1014 Flush 0 Done Done Sent false
73832024-01-24T19:08:37.782Z 16 Acked 1015 Write 8 Done Done Sent false
73842024-01-24T19:08:37.782Z 17 Acked 1016 Flush 0 Done Done Sent false
73852024-01-24T19:08:37.782Z 18 Acked 1017 Write 3 Done Done Sent false
73862024-01-24T19:08:37.782Z 19 Acked 1018 Read 3 Done Done Sent false
73872024-01-24T19:08:37.782Z 20 Acked 1019 Read 10 Done Done Sent false
73882024-01-24T19:08:37.783Z 21 Acked 1020 Read 5 Done Done Sent false
73892024-01-24T19:08:37.783Z 22 Acked 1021 Read 2 Done Done Sent false
73902024-01-24T19:08:37.783Z 23 Acked 1022 Write 3 Done Done Sent false
73912024-01-24T19:08:37.783Z 24 Acked 1023 Write 6 Done Done Sent false
73922024-01-24T19:08:37.783Z 25 Acked 1024 Write 5 Done Done Sent false
73932024-01-24T19:08:37.783Z 26 Acked 1025 Read 4 Done Done Sent false
73942024-01-24T19:08:37.784Z 27 Acked 1026 Read 1 Done Done Sent false
73952024-01-24T19:08:37.784Z 28 Acked 1027 Read 6 Done Done Sent false
73962024-01-24T19:08:37.784Z 29 Acked 1028 Read 8 Done Done Sent false
73972024-01-24T19:08:37.784Z 30 Acked 1029 Write 2 Sent Sent Sent false
73982024-01-24T19:08:37.784Z STATES DS:0 DS:1 DS:2 TOTAL
73992024-01-24T19:08:37.784Z New 0 0 0 0
74002024-01-24T19:08:37.784Z Sent 1 1 30 32
74012024-01-24T19:08:37.785Z Done 29 29 0 58
74022024-01-24T19:08:37.785Z Skipped 0 0 0 0
74032024-01-24T19:08:37.785Z Error 0 0 0 0
74042024-01-24T19:08:37.785ZLast Flush: 1016 1016 0
74052024-01-24T19:08:37.785ZDownstairs last five completed:
74062024-01-24T19:08:37.785ZUpstairs last five completed: 30 29 28 27 26
74072024-01-24T19:08:37.785ZWrote out file "/tmp/test_repair_verify.data"
74082024-01-24T19:08:37.785Z
74092024-01-24T19:08:37.785ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74102024-01-24T19:08:37.785ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74112024-01-24T19:08:37.785Z 0 000-019 62 62 62 102 102 102 F F F
74122024-01-24T19:08:37.786Z 1 020-039 60 60 60 94 94 94 F F F
74132024-01-24T19:08:37.786Z 2 040-059 68 68 66 113 113 112 F F F <---
74142024-01-24T19:08:37.786Z 3 060-079 66 66 66 112 112 112 F F F
74152024-01-24T19:08:37.786Z 4 080-099 60 60 60 95 95 95 T T F <---
74162024-01-24T19:08:37.786Z 5 100-119 63 63 63 104 104 104 F F F
74172024-01-24T19:08:37.786Z 6 120-139 63 63 63 104 104 104 F F F
74182024-01-24T19:08:37.786Z 7 140-159 66 66 66 112 112 112 F F F
74192024-01-24T19:08:37.786Z 8 160-179 68 68 66 114 114 112 F F F <---
74202024-01-24T19:08:37.786Z 9 180-199 56 56 56 89 89 89 F F F
74212024-01-24T19:08:37.786Z 10 200-219 68 68 62 113 113 103 F F F <---
74222024-01-24T19:08:37.786Z 11 220-239 56 56 56 89 89 89 F F F
74232024-01-24T19:08:37.786Z 12 240-259 68 68 65 113 113 111 F F F <---
74242024-01-24T19:08:37.786Z 13 260-279 68 68 62 113 113 103 F F F <---
74252024-01-24T19:08:37.786Z 14 280-299 53 53 53 86 86 86 F F F
74262024-01-24T19:08:37.786Z 15 300-319 66 66 66 112 112 112 F F F
74272024-01-24T19:08:37.789Z 16 320-339 64 64 64 110 110 110 F F F
74282024-01-24T19:08:37.789Z 17 340-359 66 66 66 112 112 112 F F F
74292024-01-24T19:08:37.789Z 18 360-379 66 66 66 112 112 112 F F F
74302024-01-24T19:08:37.789Z 19 380-399 64 64 64 110 110 110 F F F
74312024-01-24T19:08:37.789Z 20 400-419 64 64 64 107 107 107 F F F
74322024-01-24T19:08:37.789Z 21 420-439 68 68 62 113 113 103 F F F <---
74332024-01-24T19:08:37.789Z 22 440-459 68 68 66 113 113 112 F F F <---
74342024-01-24T19:08:37.789Z 23 460-479 66 66 66 112 112 112 T T F <---
74352024-01-24T19:08:37.789Z 24 480-499 66 66 66 112 112 112 F F F
74362024-01-24T19:08:37.789Z 25 500-519 62 62 62 103 103 103 T T F <---
74372024-01-24T19:08:37.790Z 26 520-539 66 66 66 112 112 112 T T F <---
74382024-01-24T19:08:37.790Z 27 540-559 66 66 66 112 112 112 F F F
74392024-01-24T19:08:37.790Z 28 560-579 65 65 65 111 111 111 F F F
74402024-01-24T19:08:37.790Z 29 580-599 59 59 59 92 92 92 T T F <---
74412024-01-24T19:08:37.790ZMax gen: 68, Max flush: 114
74422024-01-24T19:08:37.790ZError: Difference in extent metadata found!
74432024-01-24T19:08:37.790ZA Difference in extent metadata is expected here
74442024-01-24T19:08:37.790ZOn loop 34
74452024-01-24T19:08:37.790Z
74462024-01-24T19:08:37.790ZVerifying data now
74472024-01-24T19:08:42.332ZNo change after verify
74482024-01-24T19:08:42.332ZLoop: 34 Downstairs dump after verify (and repair):
74492024-01-24T19:08:42.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74502024-01-24T19:08:42.362Z 0 000-019 62 62 62 102 102 102 F F F
74512024-01-24T19:08:42.371Z 1 020-039 60 60 60 94 94 94 F F F
74522024-01-24T19:08:42.371Z 2 040-059 68 68 68 113 113 113 F F F
74532024-01-24T19:08:42.371Z 3 060-079 66 66 66 112 112 112 F F F
74542024-01-24T19:08:42.371Z 4 080-099 69 69 69 115 115 115 F F F
74552024-01-24T19:08:42.371Z 5 100-119 63 63 63 104 104 104 F F F
74562024-01-24T19:08:42.372Z 6 120-139 63 63 63 104 104 104 F F F
74572024-01-24T19:08:42.372Z 7 140-159 66 66 66 112 112 112 F F F
74582024-01-24T19:08:42.372Z 8 160-179 68 68 68 114 114 114 F F F
74592024-01-24T19:08:42.372Z 9 180-199 56 56 56 89 89 89 F F F
74602024-01-24T19:08:42.372Z 10 200-219 68 68 68 113 113 113 F F F
74612024-01-24T19:08:42.373Z 11 220-239 56 56 56 89 89 89 F F F
74622024-01-24T19:08:42.373Z 12 240-259 68 68 68 113 113 113 F F F
74632024-01-24T19:08:42.373Z 13 260-279 68 68 68 113 113 113 F F F
74642024-01-24T19:08:42.373Z 14 280-299 53 53 53 86 86 86 F F F
74652024-01-24T19:08:42.373Z 15 300-319 66 66 66 112 112 112 F F F
74662024-01-24T19:08:42.373Z 16 320-339 64 64 64 110 110 110 F F F
74672024-01-24T19:08:42.374Z 17 340-359 66 66 66 112 112 112 F F F
74682024-01-24T19:08:42.374Z 18 360-379 66 66 66 112 112 112 F F F
74692024-01-24T19:08:42.374Z 19 380-399 64 64 64 110 110 110 F F F
74702024-01-24T19:08:42.374Z 20 400-419 64 64 64 107 107 107 F F F
74712024-01-24T19:08:42.374Z 21 420-439 68 68 68 113 113 113 F F F
74722024-01-24T19:08:42.374Z 22 440-459 68 68 68 113 113 113 F F F
74732024-01-24T19:08:42.375Z 23 460-479 69 69 69 115 115 115 F F F
74742024-01-24T19:08:42.375Z 24 480-499 66 66 66 112 112 112 F F F
74752024-01-24T19:08:42.376Z 25 500-519 69 69 69 115 115 115 F F F
74762024-01-24T19:08:42.376Z 26 520-539 69 69 69 115 115 115 F F F
74772024-01-24T19:08:42.376Z 27 540-559 66 66 66 112 112 112 F F F
74782024-01-24T19:08:42.376Z 28 560-579 65 65 65 111 111 111 F F F
74792024-01-24T19:08:42.376Z 29 580-599 69 69 69 115 115 115 F F F
74802024-01-24T19:08:42.376ZMax gen: 69, Max flush: 115
74812024-01-24T19:08:42.376Z
74822024-01-24T19:08:42.377ZBegin loop 35
74832024-01-24T19:08:42.377ZDownstairs to restart: 2
74842024-01-24T19:08:42.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.049297782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409}
74852024-01-24T19:08:42.393Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.049742051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409}
74862024-01-24T19:08:42.416Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.049854599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409}
74872024-01-24T19:08:42.416Z{"msg":"Crucible 3edb3203-ceb4-45ca-a397-67b00e715825 has session id: 2c7585c4-1339-43bd-872d-8cfeb9ee9a5a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.049899476Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74882024-01-24T19:08:42.416Z{"msg":"Upstairs opts: Upstairs UUID: 3edb3203-ceb4-45ca-a397-67b00e715825, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.049954558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74892024-01-24T19:08:42.417Z{"msg":"Crucible stats registered with UUID: 3edb3203-ceb4-45ca-a397-67b00e715825","v":0,"name":"crucible","level":30Crucible runtime is spawned
74902024-01-24T19:08:42.418Z,"time":"2024-01-24T19:08:41.050081969Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74912024-01-24T19:08:42.418Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050271655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409}
74922024-01-24T19:08:42.419Z{"msg":"3edb3203-ceb4-45ca-a397-67b00e715825 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050491355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74932024-01-24T19:08:42.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050607304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74942024-01-24T19:08:42.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050811601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74952024-01-24T19:08:42.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050859707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74962024-01-24T19:08:42.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.050894738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74972024-01-24T19:08:42.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051131018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74982024-01-24T19:08:42.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051220683Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
74992024-01-24T19:08:42.419Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051304151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75002024-01-24T19:08:42.420Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051352144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75012024-01-24T19:08:42.421Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051408567Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75022024-01-24T19:08:42.421Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05145301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75032024-01-24T19:08:42.421Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051484141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75042024-01-24T19:08:42.421Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.051523934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75052024-01-24T19:08:42.421Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.052454739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75062024-01-24T19:08:42.421Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.052517655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75072024-01-24T19:08:42.421Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.052564792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75082024-01-24T19:08:42.421Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.052611862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75092024-01-24T19:08:42.421Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.053432549Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75102024-01-24T19:08:42.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.053525396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75112024-01-24T19:08:42.422Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.053610864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75122024-01-24T19:08:42.423Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.053783217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75132024-01-24T19:08:42.423Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054592758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75142024-01-24T19:08:42.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054666249Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75152024-01-24T19:08:42.423Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054694639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75162024-01-24T19:08:42.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054739595Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75172024-01-24T19:08:42.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054902714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75182024-01-24T19:08:42.424Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054956636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75192024-01-24T19:08:42.424Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.054983175Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75202024-01-24T19:08:42.424Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.055757654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75212024-01-24T19:08:42.425Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05583659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75222024-01-24T19:08:42.425Z{"msg":"[0]R flush_numbers[0..12]: [102, 94, 113, 112, 115, 104, 104, 112, 114, 89, 113, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.055870313Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75232024-01-24T19:08:42.425Z{"msg":"[0]R generation[0..12]: [62, 60, 68, 66, 69, 63, 63, 66, 68, 56, 68, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.055939786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75242024-01-24T19:08:42.425Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.055988653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75252024-01-24T19:08:42.426Z{"msg":"[1]R flush_numbers[0..12]: [102, 94, 113, 112, 115, 104, 104, 112, 114, 89, 113, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05604073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75262024-01-24T19:08:42.426Z{"msg":"[1]R generation[0..12]: [62, 60, 68, 66, 69, 63, 63, 66, 68, 56, 68, 56]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05606792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75272024-01-24T19:08:42.426Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056091665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75282024-01-24T19:08:42.426Z{"msg":"[2]R flush_numbers[0..12]: [102, 94, 113, 112, 115, 104, 104, 112, 114, 89, 113, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056114692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75292024-01-24T19:08:42.426Z{"msg":"[2]R generation[0..12]: [62, 60, 68, 66, 69, 63, 63, 66, 68, 56, 68, 56]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
75302024-01-24T19:08:42.427Z,"time":"2024-01-24T19:08:41.056158526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75312024-01-24T19:08:42.427Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056188686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75322024-01-24T19:08:42.427Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05623128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75332024-01-24T19:08:42.427Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056256689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75342024-01-24T19:08:42.427Z{"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056278152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75352024-01-24T19:08:42.427Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056320171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75362024-01-24T19:08:42.427Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056345842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75372024-01-24T19:08:42.427Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056365882Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75382024-01-24T19:08:42.427Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056408168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75392024-01-24T19:08:42.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.05643271Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75402024-01-24T19:08:42.428Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056475777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75412024-01-24T19:08:42.429Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056499647Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75422024-01-24T19:08:42.429Z{"msg":"3edb3203-ceb4-45ca-a397-67b00e715825 is now active with session: 2c7585c4-1339-43bd-872d-8cfeb9ee9a5a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056547288Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75432024-01-24T19:08:42.429Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056570227Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409,"session_id":"2c7585c4-1339-43bd-872d-8cfeb9ee9a5a"}
75442024-01-24T19:08:42.429Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:41.056611146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1409}
75452024-01-24T19:08:42.429ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75462024-01-24T19:08:42.429ZLoading write count information from file "/tmp/test_repair_verify.data"
75472024-01-24T19:08:42.429ZRun Repair workload
75482024-01-24T19:08:42.430Z01/30 Write block 230 len 24576 data: 5 6 6 6 6 4
75492024-01-24T19:08:42.430Z02/30 Read block 235 len 36864
75502024-01-24T19:08:42.848Z03/30 Write block 83 len 24576 data: 11 10 8 9 7 6
75512024-01-24T19:08:42.848Z04/30 Write block 273 len 12288 data: 7 5 4
75522024-01-24T19:08:42.848Z05/30 Read block 328 len 4096
75532024-01-24T19:08:42.848Z06/30 Read block 335 len 36864
75542024-01-24T19:08:42.848Z07/30 Write block 186 len 28672 data: 7 6 6 5 4 4 4
75552024-01-24T19:08:42.849Z08/30 Read block 208 len 12288
75562024-01-24T19:08:42.849Z09/30 Read block 181 len 4096
75572024-01-24T19:08:42.849Z10/30 Write block 6 len 40960 data: 5 6 7 5 5 4 4 4 3 5
75582024-01-24T19:08:42.849Z11/30 Read block 521 len 12288
75592024-01-24T19:08:42.849Z12/30 Write block 480 len 20480 data: 8 10 10 6 5
75602024-01-24T19:08:42.850Z13/30 Read block 520 len 20480
75612024-01-24T19:08:42.850Z14/30 Read block 31 len 16384
75622024-01-24T19:08:42.851Z15/30 Read block 569 len 24576
75632024-01-24T19:08:42.851Z16/30 Read block 465 len 36864
75642024-01-24T19:08:42.851Z17/30 Read block 300 len 24576
75652024-01-24T19:08:42.851Z18/30 Read block 331 len 36864
75662024-01-24T19:08:42.851Z19/30 Read block 564 len 32768
75672024-01-24T19:08:42.851Z20/30 Write block 1 len 40960 data: 3 3 4 5 5 6 7 8 6 6
75682024-01-24T19:08:42.851Z21/30 Write block 91 len 16384 data: 6 7 8 9
75692024-01-24T19:08:42.852Z22/30 Flush
75702024-01-24T19:08:42.852Z23/30 Write block 497 len 24576 data: 7 7 6 5 5 3
75712024-01-24T19:08:42.852Z24/30 Read block 115 len 8192
75722024-01-24T19:08:42.852Z25/30 Write block 456 len 12288 data: 7 7 7
75732024-01-24T19:08:42.868Z26/30 Read block 245 len 12288
75742024-01-24T19:08:42.868Z27/30 Write block 184 len 12288 data: 7 8 8
75752024-01-24T19:08:42.868Z28/30 Write block 528 len 8192 data: 8 7
75762024-01-24T19:08:42.869Z29/30 Write block 78 len 40960 data: 8 9 9 11 10 12 11 9 10 8
75772024-01-24T19:08:42.869Z30/30 Read block 422 len 20480
75782024-01-24T19:08:42.869Z----------------------------------------------------------------
75792024-01-24T19:08:42.869Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
75802024-01-24T19:08:42.869ZStates: Active Active Active
75812024-01-24T19:08:42.869ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
75822024-01-24T19:08:42.869Z 1 Acked 1000 Write 6 Done Done Done false
75832024-01-24T19:08:42.869Z 2 Acked 1001 Read 9 Done Done Done false
75842024-01-24T19:08:42.870Z 3 Acked 1002 Write 6 Done Done Done false
75852024-01-24T19:08:42.871Z 4 Acked 1003 Write 3 Done Done Done false
75862024-01-24T19:08:42.871Z 5 Acked 1004 Read 1 Done Done Sent false
75872024-01-24T19:08:42.871Z 6 Acked 1005 Read 9 Done Done Sent false
75882024-01-24T19:08:42.872Z 7 Acked 1006 Write 7 Done Done Sent false
75892024-01-24T19:08:42.872Z 8 Acked 1007 Read 3 Done Done Sent false
75902024-01-24T19:08:42.872Z 9 Acked 1008 Read 1 Done Done Sent false
75912024-01-24T19:08:42.872Z 10 Acked 1009 Write 10 Done Done Sent false
75922024-01-24T19:08:42.872Z 11 Acked 1010 Read 3 Done Done Sent false
75932024-01-24T19:08:42.872Z 12 Acked 1011 Write 5 Done Done Sent false
75942024-01-24T19:08:42.872Z 13 Acked 1012 Read 5 Done Done Sent false
75952024-01-24T19:08:42.872Z 14 Acked 1013 Read 4 Done Done Sent false
75962024-01-24T19:08:42.873Z 15 Acked 1014 Read 6 Done Done Sent false
75972024-01-24T19:08:42.873Z 16 Acked 1015 Read 9 Done Done Sent false
75982024-01-24T19:08:42.873Z 17 Acked 1016 Read 6 Done Done Sent false
75992024-01-24T19:08:42.873Z 18 Acked 1017 Read 9 Done Done Sent false
76002024-01-24T19:08:42.873Z 19 Acked 1018 Read 8 Done Done Sent false
76012024-01-24T19:08:42.873Z 20 Acked 1019 Write 10 Done Done Sent false
76022024-01-24T19:08:42.873Z 21 Acked 1020 Write 4 Done Done Sent false
76032024-01-24T19:08:42.873Z 22 Acked 1021 Flush 0 Done Done Sent false
76042024-01-24T19:08:42.873Z 23 Acked 1022 Write 6 Done Done Sent false
76052024-01-24T19:08:42.873Z 24 Acked 1023 Read 2 Done Done Sent false
76062024-01-24T19:08:42.873Z 25 Acked 1024 Write 3 Done Done Sent false
76072024-01-24T19:08:42.873Z 26 Acked 1025 Read 3 Done Done Sent false
76082024-01-24T19:08:42.874Z 27 Acked 1026 Write 3 Done Done Sent false
76092024-01-24T19:08:42.874Z 28 Acked 1027 Write 2 Done Done Sent false
76102024-01-24T19:08:42.874Z 29 Acked 1028 Write 10 Done Done Sent false
76112024-01-24T19:08:42.874Z 30 Acked 1029 Read 5 Sent Done Sent false
76122024-01-24T19:08:42.874Z STATES DS:0 DS:1 DS:2 TOTAL
76132024-01-24T19:08:42.875Z New 0 0 0 0
76142024-01-24T19:08:42.875Z Sent 1 0 26 27
76152024-01-24T19:08:42.875Z Done 29 30 4 63
76162024-01-24T19:08:42.875Z Skipped 0 0 0 0
76172024-01-24T19:08:42.875Z Error 0 0 0 0
76182024-01-24T19:08:42.875ZLast Flush: 1021 1021 0
76192024-01-24T19:08:42.875ZDownstairs last five completed:
76202024-01-24T19:08:42.875ZUpstairs last five completed: 30 29 28 27 26
76212024-01-24T19:08:42.876ZWrote out file "/tmp/test_repair_verify.data"
76222024-01-24T19:08:42.876Z
76232024-01-24T19:08:42.876ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76242024-01-24T19:08:42.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76252024-01-24T19:08:42.876Z 0 000-019 70 70 62 116 116 102 F F F <---
76262024-01-24T19:08:42.876Z 1 020-039 60 60 60 94 94 94 F F F
76272024-01-24T19:08:42.876Z 2 040-059 68 68 68 113 113 113 F F F
76282024-01-24T19:08:42.876Z 3 060-079 66 66 66 112 112 112 T T F <---
76292024-01-24T19:08:42.877Z 4 080-099 70 70 69 116 116 115 T T T <---
76302024-01-24T19:08:42.877Z 5 100-119 63 63 63 104 104 104 F F F
76312024-01-24T19:08:42.877Z 6 120-139 63 63 63 104 104 104 F F F
76322024-01-24T19:08:42.877Z 7 140-159 66 66 66 112 112 112 F F F
76332024-01-24T19:08:42.878Z 8 160-179 68 68 68 114 114 114 F F F
76342024-01-24T19:08:42.878Z 9 180-199 70 70 56 116 116 89 T T F <---
76352024-01-24T19:08:42.878Z 10 200-219 68 68 68 113 113 113 F F F
76362024-01-24T19:08:42.878Z 11 220-239 70 70 56 116 116 89 F F T <---
76372024-01-24T19:08:42.879Z 12 240-259 68 68 68 113 113 113 F F F
76382024-01-24T19:08:42.879Z 13 260-279 70 70 68 116 116 113 F F T <---
76392024-01-24T19:08:42.879Z 14 280-299 53 53 53 86 86 86 F F F
76402024-01-24T19:08:42.879Z 15 300-319 66 66 66 112 112 112 F F F
76412024-01-24T19:08:42.879Z 16 320-339 64 64 64 110 110 110 F F F
76422024-01-24T19:08:42.879Z 17 340-359 66 66 66 112 112 112 F F F
76432024-01-24T19:08:42.879Z 18 360-379 66 66 66 112 112 112 F F F
76442024-01-24T19:08:42.880Z 19 380-399 64 64 64 110 110 110 F F F
76452024-01-24T19:08:42.880Z 20 400-419 64 64 64 107 107 107 F F F
76462024-01-24T19:08:42.880Z 21 420-439 68 68 68 113 113 113 F F F
76472024-01-24T19:08:42.880Z 22 440-459 68 68 68 113 113 113 T T F <---
76482024-01-24T19:08:42.880Z 23 460-479 69 69 69 115 115 115 F F F
76492024-01-24T19:08:42.880Z 24 480-499 70 70 66 116 116 112 T T F <---
76502024-01-24T19:08:42.880Z 25 500-519 69 69 69 115 115 115 T T F <---
76512024-01-24T19:08:42.881Z 26 520-539 69 69 69 115 115 115 T T F <---
76522024-01-24T19:08:42.881Z 27 540-559 66 66 66 112 112 112 F F F
76532024-01-24T19:08:42.881Z 28 560-579 65 65 65 111 111 111 F F F
76542024-01-24T19:08:42.881Z 29 580-599 69 69 69 115 115 115 F F F
76552024-01-24T19:08:42.881ZMax gen: 70, Max flush: 116
76562024-01-24T19:08:42.881ZError: Difference in extent metadata found!
76572024-01-24T19:08:42.882ZA Difference in extent metadata is expected here
76582024-01-24T19:08:42.883ZOn loop 35
76592024-01-24T19:08:42.883Z
76602024-01-24T19:08:42.883ZVerifying data now
76612024-01-24T19:08:48.473ZNo change after verify
76622024-01-24T19:08:48.474ZLoop: 35 Downstairs dump after verify (and repair):
76632024-01-24T19:08:48.491ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76642024-01-24T19:08:48.491Z 0 000-019 70 70 70 116 116 116 F F F
76652024-01-24T19:08:48.492Z 1 020-039 60 60 60 94 94 94 F F F
76662024-01-24T19:08:48.492Z 2 040-059 68 68 68 113 113 113 F F F
76672024-01-24T19:08:48.492Z 3 060-079 71 71 71 117 117 117 F F F
76682024-01-24T19:08:48.492Z 4 080-099 71 71 71 117 117 117 F F F
76692024-01-24T19:08:48.492Z 5 100-119 63 63 63 104 104 104 F F F
76702024-01-24T19:08:48.492Z 6 120-139 63 63 63 104 104 104 F F F
76712024-01-24T19:08:48.492Z 7 140-159 66 66 66 112 112 112 F F F
76722024-01-24T19:08:48.492Z 8 160-179 68 68 68 114 114 114 F F F
76732024-01-24T19:08:48.492Z 9 180-199 71 71 71 117 117 117 F F F
76742024-01-24T19:08:48.492Z 10 200-219 68 68 68 113 113 113 F F F
76752024-01-24T19:08:48.492Z 11 220-239 70 70 70 116 116 116 F F F
76762024-01-24T19:08:48.492Z 12 240-259 68 68 68 113 113 113 F F F
76772024-01-24T19:08:48.492Z 13 260-279 70 70 70 116 116 116 F F F
76782024-01-24T19:08:48.492Z 14 280-299 53 53 53 86 86 86 F F F
76792024-01-24T19:08:48.493Z 15 300-319 66 66 66 112 112 112 F F F
76802024-01-24T19:08:48.493Z 16 320-339 64 64 64 110 110 110 F F F
76812024-01-24T19:08:48.493Z 17 340-359 66 66 66 112 112 112 F F F
76822024-01-24T19:08:48.493Z 18 360-379 66 66 66 112 112 112 F F F
76832024-01-24T19:08:48.494Z 19 380-399 64 64 64 110 110 110 F F F
76842024-01-24T19:08:48.494Z 20 400-419 64 64 64 107 107 107 F F F
76852024-01-24T19:08:48.494Z 21 420-439 68 68 68 113 113 113 F F F
76862024-01-24T19:08:48.494Z 22 440-459 71 71 71 117 117 117 F F F
76872024-01-24T19:08:48.494Z 23 460-479 69 69 69 115 115 115 F F F
76882024-01-24T19:08:48.494Z 24 480-499 71 71 71 117 117 117 F F F
76892024-01-24T19:08:48.494Z 25 500-519 71 71 71 117 117 117 F F F
76902024-01-24T19:08:48.494Z 26 520-539 71 71 71 117 117 117 F F F
76912024-01-24T19:08:48.495Z 27 540-559 66 66 66 112 112 112 F F F
76922024-01-24T19:08:48.495Z 28 560-579 65 65 65 111 111 111 F F F
76932024-01-24T19:08:48.495Z 29 580-599 69 69 69 115 115 115 F F F
76942024-01-24T19:08:48.495ZMax gen: 71, Max flush: 117
76952024-01-24T19:08:48.495Z
76962024-01-24T19:08:48.495ZBegin loop 36
76972024-01-24T19:08:48.495ZDownstairs to restart: 2
76982024-01-24T19:08:48.521Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.152527561Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417}
76992024-01-24T19:08:48.522Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153015173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417}
77002024-01-24T19:08:48.522Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153046041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417}
77012024-01-24T19:08:48.522Z{"msg":"Crucible b22572f4-1a19-48ff-bafe-02668d0a6def has session id: 9f0cd270-044d-4a8c-8e6e-92d07533d1c8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153067411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77022024-01-24T19:08:48.523Z{"msg":"Upstairs opts: Upstairs UUID: b22572f4-1a19-48ff-bafe-02668d0a6def, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.15310263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77032024-01-24T19:08:48.524Z{"msg":"Crucible stats registered with UUID: b22572f4-1a19-48ff-bafe-02668d0a6def","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153159209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77042024-01-24T19:08:48.524ZCrucible runtime is spawned
77052024-01-24T19:08:48.524Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153655011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417}
77062024-01-24T19:08:48.524Z{"msg":"b22572f4-1a19-48ff-bafe-02668d0a6def active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153840499Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77072024-01-24T19:08:48.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.15390054Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77082024-01-24T19:08:48.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153928624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77092024-01-24T19:08:48.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.153987526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77102024-01-24T19:08:48.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.154130836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77112024-01-24T19:08:48.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.154207115Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77122024-01-24T19:08:48.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.154250523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77132024-01-24T19:08:48.525Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.15428932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77142024-01-24T19:08:48.526Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.154532602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77152024-01-24T19:08:48.526Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.154573599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77162024-01-24T19:08:48.526Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.15473823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77172024-01-24T19:08:48.526Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.1547821Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77182024-01-24T19:08:48.526Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.1548563Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77192024-01-24T19:08:48.526Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156310195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77202024-01-24T19:08:48.527Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156375887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77212024-01-24T19:08:48.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156404077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77222024-01-24T19:08:48.531Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156485416Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77232024-01-24T19:08:48.532Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156658846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77242024-01-24T19:08:48.532Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.156712867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77252024-01-24T19:08:48.532Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.157514645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77262024-01-24T19:08:48.532Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.157652241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77272024-01-24T19:08:48.533Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.157720459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77282024-01-24T19:08:48.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158526688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77292024-01-24T19:08:48.533Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158664345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77302024-01-24T19:08:48.533Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158702088Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77312024-01-24T19:08:48.533Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158724201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77322024-01-24T19:08:48.534Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.15876915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77332024-01-24T19:08:48.535Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158795459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77342024-01-24T19:08:48.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158837226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77352024-01-24T19:08:48.535Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158862961Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77362024-01-24T19:08:48.535Z{"msg":"[0]R flush_numbers[0..12]: [116, 94, 113, 117, 117, 104, 104, 112, 114, 117, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158886521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77372024-01-24T19:08:48.535Z{"msg":"[0]R generation[0..12]: [70, 60, 68, 71, 71, 63, 63, 66, 68, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158930965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77382024-01-24T19:08:48.536Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.158958298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77392024-01-24T19:08:48.536Z{"msg":"[1]R flush_numbers[0..12]: [116, 94, 113, 117, 117, 104, 104, 112, 114, 117, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159020532Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77402024-01-24T19:08:48.536Z{"msg":"[1]R generation[0..12]: [70, 60, 68, 71, 71, 63, 63, 66, 68, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159075053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77412024-01-24T19:08:48.536Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159102609Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO
77422024-01-24T19:08:48.538Z":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77432024-01-24T19:08:48.539Z{"msg":"[2]R flush_numbers[0..12]: [116, 94, 113, 117, 117, 104, 104, 112, 114, 117, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159223053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77442024-01-24T19:08:48.539Z{"msg":"[2]R generation[0..12]: [70, 60, 68, 71, 71, 63, 63, 66, 68, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159285366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77452024-01-24T19:08:48.539Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159312583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77462024-01-24T19:08:48.539Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159366167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77472024-01-24T19:08:48.539Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159391888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77482024-01-24T19:08:48.540Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159439089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77492024-01-24T19:08:48.540Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159463878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77502024-01-24T19:08:48.540Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159506512Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77512024-01-24T19:08:48.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159529559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77522024-01-24T19:08:48.542Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159572402Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77532024-01-24T19:08:48.543Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159597364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77542024-01-24T19:08:48.543Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159648992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77552024-01-24T19:08:48.543Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159673727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77562024-01-24T19:08:48.543Z{"msg":"b22572f4-1a19-48ff-bafe-02668d0a6def is now active with session: 9f0cd270-044d-4a8c-8e6e-92d07533d1c8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159717795Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77572024-01-24T19:08:48.544Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159740938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417,"session_id":"9f0cd270-044d-4a8c-8e6e-92d07533d1c8"}
77582024-01-24T19:08:48.544Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:47.159782428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1417}
77592024-01-24T19:08:48.544ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77602024-01-24T19:08:48.544ZLoading write count information from file "/tmp/test_repair_verify.data"
77612024-01-24T19:08:48.544ZRun Repair workload
77622024-01-24T19:08:48.545Z01/30 Write block 511 len 8192 data: 9 9
77632024-01-24T19:08:48.545Z02/30 Read block 301 len 28672
77642024-01-24T19:08:48.551Z03/30 Read block 277 len 12288
77652024-01-24T19:08:48.972Z04/30 Read block 251 len 40960
77662024-01-24T19:08:48.972Z05/30 Write block 578 len 4096 data: 5
77672024-01-24T19:08:48.972Z06/30 Write block 103 len 8192 data: 7 7
77682024-01-24T19:08:48.972Z07/30 Read block 398 len 24576
77692024-01-24T19:08:48.972Z08/30 Write block 331 len 36864 data: 5 7 7 6 6 6 8 8 8
77702024-01-24T19:08:48.972Z09/30 Read block 510 len 8192
77712024-01-24T19:08:48.972Z10/30 Write block 293 len 8192 data: 8 9
77722024-01-24T19:08:48.972Z11/30 Read block 56 len 20480
77732024-01-24T19:08:48.973Z12/30 Write block 190 len 40960 data: 5 5 5 5 5 4 4 4 4 3
77742024-01-24T19:08:48.973Z13/30 Write block 91 len 24576 data: 7 8 9 10 8 8
77752024-01-24T19:08:48.973Z14/30 Read block 523 len 40960
77762024-01-24T19:08:48.973Z15/30 Read block 495 len 20480
77772024-01-24T19:08:48.973Z16/30 Write block 524 len 36864 data: 8 8 8 8 9 8 7 5 5
77782024-01-24T19:08:48.973Z17/30 Write block 272 len 28672 data: 7 8 6 5 4 7 8
77792024-01-24T19:08:48.973Z18/30 Read block 43 len 24576
77802024-01-24T19:08:48.973Z19/30 Read block 390 len 20480
77812024-01-24T19:08:48.973Z20/30 Read block 317 len 40960
77822024-01-24T19:08:48.973Z21/30 Write block 270 len 16384 data: 7 7 8 9
77832024-01-24T19:08:48.973Z22/30 Write block 596 len 12288 data: 6 4 4
77842024-01-24T19:08:48.973Z23/30 Read block 513 len 28672
77852024-01-24T19:08:48.973Z24/30 Read block 241 len 32768
77862024-01-24T19:08:48.973Z25/30 Read block 210 len 4096
77872024-01-24T19:08:48.973Z26/30 Read block 231 len 32768
77882024-01-24T19:08:48.974Z27/30 Write block 508 len 20480 data: 6 6 8 10 10
77892024-01-24T19:08:48.974Z28/30 Read block 220 len 36864
77902024-01-24T19:08:48.974Z29/30 Write block 571 len 24576 data: 4 5 4 5 6 6
77912024-01-24T19:08:48.974Z30/30 Write block 295 len 24576 data: 8 7 5 5 4 4
77922024-01-24T19:08:48.974Z----------------------------------------------------------------
77932024-01-24T19:08:48.974Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
77942024-01-24T19:08:48.974ZStates: Active Active Active
77952024-01-24T19:08:48.974ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
77962024-01-24T19:08:48.976Z 1 Acked 1000 Write 2 Done Done Sent false
77972024-01-24T19:08:48.976Z 2 Acked 1001 Read 7 Done Done Sent false
77982024-01-24T19:08:48.976Z 3 Acked 1002 Read 3 Done Done Sent false
77992024-01-24T19:08:48.976Z 4 Acked 1003 Read 10 Done Done Sent false
78002024-01-24T19:08:48.976Z 5 Acked 1004 Write 1 Done Done Sent false
78012024-01-24T19:08:48.976Z 6 Acked 1005 Write 2 Done Done Sent false
78022024-01-24T19:08:48.976Z 7 Acked 1006 Read 6 Done Done Sent false
78032024-01-24T19:08:48.976Z 8 Acked 1007 Write 9 Done Done Sent false
78042024-01-24T19:08:48.976Z 9 Acked 1008 Read 2 Done Done Sent false
78052024-01-24T19:08:48.976Z 10 Acked 1009 Write 2 Done Done Sent false
78062024-01-24T19:08:48.976Z 11 Acked 1010 Read 5 Done Done Sent false
78072024-01-24T19:08:48.976Z 12 Acked 1011 Write 10 Done Done Sent false
78082024-01-24T19:08:48.976Z 13 Acked 1012 Write 6 Done Done Sent false
78092024-01-24T19:08:48.976Z 14 Acked 1013 Read 10 Done Done Sent false
78102024-01-24T19:08:48.976Z 15 Acked 1014 Read 5 Done Done Sent false
78112024-01-24T19:08:48.976Z 16 Acked 1015 Write 9 Done Done Sent false
78122024-01-24T19:08:48.976Z 17 Acked 1016 Write 7 Done Done Sent false
78132024-01-24T19:08:48.976Z 18 Acked 1017 Read 6 Done Done Sent false
78142024-01-24T19:08:48.976Z 19 Acked 1018 Read 5 Done Done Sent false
78152024-01-24T19:08:48.976Z 20 Acked 1019 Read 10 Done Done Sent false
78162024-01-24T19:08:48.977Z 21 Acked 1020 Write 4 Done Done Sent false
78172024-01-24T19:08:48.977Z 22 Acked 1021 Write 3 Done Done Sent false
78182024-01-24T19:08:48.977Z 23 Acked 1022 Read 7 Done Done Sent false
78192024-01-24T19:08:48.978Z 24 Acked 1023 Read 8 Done Done Sent false
78202024-01-24T19:08:48.978Z 25 Acked 1024 Read 1 Done Done Sent false
78212024-01-24T19:08:48.978Z 26 Acked 1025 Read 8 Done Done Sent false
78222024-01-24T19:08:48.978Z 27 Acked 1026 Write 5 Done Done Sent false
78232024-01-24T19:08:48.978Z 28 Acked 1027 Read 9 Done Done Sent false
78242024-01-24T19:08:48.978Z 29 Acked 1028 Write 6 Done Done Sent false
78252024-01-24T19:08:48.978Z 30 Acked 1029 Write 6 Sent Sent Sent false
78262024-01-24T19:08:48.978Z STATES DS:0 DS:1 DS:2 TOTAL
78272024-01-24T19:08:48.978Z New 0 0 0 0
78282024-01-24T19:08:48.978Z Sent 1 1 30 32
78292024-01-24T19:08:48.978Z Done 29 29 0 58
78302024-01-24T19:08:48.979Z Skipped 0 0 0 0
78312024-01-24T19:08:48.979Z Error 0 0 0 0
78322024-01-24T19:08:48.979ZLast Flush: 0 0 0
78332024-01-24T19:08:48.979ZDownstairs last five completed:
78342024-01-24T19:08:48.979ZUpstairs last five completed: 30 29 28 27 26
78352024-01-24T19:08:48.979ZWrote out file "/tmp/test_repair_verify.data"
78362024-01-24T19:08:48.979Z
78372024-01-24T19:08:48.979ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78382024-01-24T19:08:48.984ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78392024-01-24T19:08:48.984Z 0 000-019 70 70 70 116 116 116 F F F
78402024-01-24T19:08:48.984Z 1 020-039 60 60 60 94 94 94 F F F
78412024-01-24T19:08:48.984Z 2 040-059 68 68 68 113 113 113 F F F
78422024-01-24T19:08:48.984Z 3 060-079 71 71 71 117 117 117 F F F
78432024-01-24T19:08:48.985Z 4 080-099 71 71 71 117 117 117 T T F <---
78442024-01-24T19:08:48.985Z 5 100-119 63 63 63 104 104 104 T T F <---
78452024-01-24T19:08:48.985Z 6 120-139 63 63 63 104 104 104 F F F
78462024-01-24T19:08:48.985Z 7 140-159 66 66 66 112 112 112 F F F
78472024-01-24T19:08:48.985Z 8 160-179 68 68 68 114 114 114 F F F
78482024-01-24T19:08:48.985Z 9 180-199 71 71 71 117 117 117 T T F <---
78492024-01-24T19:08:48.985Z 10 200-219 68 68 68 113 113 113 F F F
78502024-01-24T19:08:48.985Z 11 220-239 70 70 70 116 116 116 F F F
78512024-01-24T19:08:48.985Z 12 240-259 68 68 68 113 113 113 F F F
78522024-01-24T19:08:48.985Z 13 260-279 70 70 70 116 116 116 T T F <---
78532024-01-24T19:08:48.985Z 14 280-299 53 53 53 86 86 86 T T F <---
78542024-01-24T19:08:48.985Z 15 300-319 66 66 66 112 112 112 T T F <---
78552024-01-24T19:08:48.985Z 16 320-339 64 64 64 110 110 110 T T F <---
78562024-01-24T19:08:48.986Z 17 340-359 66 66 66 112 112 112 F F F
78572024-01-24T19:08:48.986Z 18 360-379 66 66 66 112 112 112 F F F
78582024-01-24T19:08:48.986Z 19 380-399 64 64 64 110 110 110 F F F
78592024-01-24T19:08:48.986Z 20 400-419 64 64 64 107 107 107 F F F
78602024-01-24T19:08:48.986Z 21 420-439 68 68 68 113 113 113 F F F
78612024-01-24T19:08:48.986Z 22 440-459 71 71 71 117 117 117 F F F
78622024-01-24T19:08:48.986Z 23 460-479 69 69 69 115 115 115 F F F
78632024-01-24T19:08:48.986Z 24 480-499 71 71 71 117 117 117 F F F
78642024-01-24T19:08:48.986Z 25 500-519 71 71 71 117 117 117 T T F <---
78652024-01-24T19:08:48.987Z 26 520-539 71 71 71 117 117 117 T T F <---
78662024-01-24T19:08:48.987Z 27 540-559 66 66 66 112 112 112 F F F
78672024-01-24T19:08:48.988Z 28 560-579 65 65 65 111 111 111 T T F <---
78682024-01-24T19:08:48.988Z 29 580-599 69 69 69 115 115 115 T T F <---
78692024-01-24T19:08:48.988ZMax gen: 71, Max flush: 117
78702024-01-24T19:08:48.988ZError: Difference in extent metadata found!
78712024-01-24T19:08:48.988ZA Difference in extent metadata is expected here
78722024-01-24T19:08:48.988ZOn loop 36
78732024-01-24T19:08:48.988Z
78742024-01-24T19:08:48.988ZVerifying data now
78752024-01-24T19:08:53.616ZNo change after verify
78762024-01-24T19:08:53.616ZLoop: 36 Downstairs dump after verify (and repair):
78772024-01-24T19:08:53.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78782024-01-24T19:08:53.684Z 0 000-019 70 70 70 116 116 116 F F F
78792024-01-24T19:08:53.684Z 1 020-039 60 60 60 94 94 94 F F F
78802024-01-24T19:08:53.685Z 2 040-059 68 68 68 113 113 113 F F F
78812024-01-24T19:08:53.685Z 3 060-079 71 71 71 117 117 117 F F F
78822024-01-24T19:08:53.685Z 4 080-099 72 72 72 118 118 118 F F F
78832024-01-24T19:08:53.685Z 5 100-119 72 72 72 118 118 118 F F F
78842024-01-24T19:08:53.703Z 6 120-139 63 63 63 104 104 104 F F F
78852024-01-24T19:08:53.703Z 7 140-159 66 66 66 112 112 112 F F F
78862024-01-24T19:08:53.703Z 8 160-179 68 68 68 114 114 114 F F F
78872024-01-24T19:08:53.703Z 9 180-199 72 72 72 118 118 118 F F F
78882024-01-24T19:08:53.703Z 10 200-219 68 68 68 113 113 113 F F F
78892024-01-24T19:08:53.703Z 11 220-239 70 70 70 116 116 116 F F F
78902024-01-24T19:08:53.704Z 12 240-259 68 68 68 113 113 113 F F F
78912024-01-24T19:08:53.704Z 13 260-279 72 72 72 118 118 118 F F F
78922024-01-24T19:08:53.704Z 14 280-299 72 72 72 118 118 118 F F F
78932024-01-24T19:08:53.704Z 15 300-319 72 72 72 118 118 118 F F F
78942024-01-24T19:08:53.704Z 16 320-339 72 72 72 118 118 118 F F F
78952024-01-24T19:08:53.704Z 17 340-359 66 66 66 112 112 112 F F F
78962024-01-24T19:08:53.704Z 18 360-379 66 66 66 112 112 112 F F F
78972024-01-24T19:08:53.704Z 19 380-399 64 64 64 110 110 110 F F F
78982024-01-24T19:08:53.704Z 20 400-419 64 64 64 107 107 107 F F F
78992024-01-24T19:08:53.704Z 21 420-439 68 68 68 113 113 113 F F F
79002024-01-24T19:08:53.705Z 22 440-459 71 71 71 117 117 117 F F F
79012024-01-24T19:08:53.705Z 23 460-479 69 69 69 115 115 115 F F F
79022024-01-24T19:08:53.705Z 24 480-499 71 71 71 117 117 117 F F F
79032024-01-24T19:08:53.705Z 25 500-519 72 72 72 118 118 118 F F F
79042024-01-24T19:08:53.705Z 26 520-539 72 72 72 118 118 118 F F F
79052024-01-24T19:08:53.705Z 27 540-559 66 66 66 112 112 112 F F F
79062024-01-24T19:08:53.705Z 28 560-579 72 72 72 118 118 118 F F F
79072024-01-24T19:08:53.705Z 29 580-599 72 72 72 118 118 118 F F F
79082024-01-24T19:08:53.705ZMax gen: 72, Max flush: 118
79092024-01-24T19:08:53.705Z
79102024-01-24T19:08:53.706ZBegin loop 37
79112024-01-24T19:08:53.706ZDownstairs to restart: 0
79122024-01-24T19:08:53.706Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.339331868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425}
79132024-01-24T19:08:53.706Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.339829633Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425}
79142024-01-24T19:08:53.707Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.339860393Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425}
79152024-01-24T19:08:53.707Z{"msg":"Crucible b04f7443-49ae-4ffb-b245-8ed384bf6cd0 has session id: eb93ce3b-2e90-4908-96a4-141ea8b87c9e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.33992471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79162024-01-24T19:08:53.707Z{"msg":"Upstairs opts: Upstairs UUID: b04f7443-49ae-4ffb-b245-8ed384bf6cd0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.339970762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79172024-01-24T19:08:53.707ZCrucible runtime is spawned
79182024-01-24T19:08:53.707Z{"msg":"Crucible stats registered with UUID: b04f7443-49ae-4ffb-b245-8ed384bf6cd0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340162533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79192024-01-24T19:08:53.708Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340249602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425}
79202024-01-24T19:08:53.708Z{"msg":"b04f7443-49ae-4ffb-b245-8ed384bf6cd0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340554174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79212024-01-24T19:08:53.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340672416Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79222024-01-24T19:08:53.708Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340715062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79232024-01-24T19:08:53.708Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340790524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79242024-01-24T19:08:53.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340822429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79252024-01-24T19:08:53.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340874759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79262024-01-24T19:08:53.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.340901263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79272024-01-24T19:08:53.710Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341131325Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79282024-01-24T19:08:53.710Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341363669Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79292024-01-24T19:08:53.711Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341414312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79302024-01-24T19:08:53.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341456041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79312024-01-24T19:08:53.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341483861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79322024-01-24T19:08:53.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.341565434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79332024-01-24T19:08:53.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.342769448Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79342024-01-24T19:08:53.712Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.34285599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79352024-01-24T19:08:53.712Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.342884388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79362024-01-24T19:08:53.712Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.343062399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79372024-01-24T19:08:53.712Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.343489202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79382024-01-24T19:08:53.712Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.343599891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79392024-01-24T19:08:53.712Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.344165537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79402024-01-24T19:08:53.713Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.344614194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79412024-01-24T19:08:53.713Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.344742129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79422024-01-24T19:08:53.713Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345144952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79432024-01-24T19:08:53.713Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345232238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79442024-01-24T19:08:53.713Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345307356Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79452024-01-24T19:08:53.714Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345655951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79462024-01-24T19:08:53.714Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345732044Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79472024-01-24T19:08:53.714Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345761498Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79482024-01-24T19:08:53.714Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345782623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79492024-01-24T19:08:53.714Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.34582693Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79502024-01-24T19:08:53.714Z{"msg":"[0]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 118, 104, 112, 114, 118, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345854688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79512024-01-24T19:08:53.714Z{"msg":"[0]R generation[0..12]: [70, 60, 68, 71, 72, 72, 63, 66, 68, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345899808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79522024-01-24T19:08:53.714Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345927014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79532024-01-24T19:08:53.714Z{"msg":"[1]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 118, 104, 112, 114, 118, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.345951588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79542024-01-24T19:08:53.716Z{"msg":"[1]R generation[0..12]: [70, 60, 68, 71, 72, 72, 63, 66, 68, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.3459951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79552024-01-24T19:08:53.716Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346022135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79562024-01-24T19:08:53.716Z{"msg":"[2]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 118, 104, 112, 114, 118, 113, 116]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346066816Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79572024-01-24T19:08:53.716Z{"msg":"[2]R generation[0..12]: [70, 60, 68, 71, 72, 72, 63, 66, 68, 72, 68, 70]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346092528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79582024-01-24T19:08:53.716Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
79592024-01-24T19:08:53.716Z,"time":"2024-01-24T19:08:52.34613715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79602024-01-24T19:08:53.716Z{"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346163029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79612024-01-24T19:08:53.717Z{"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346206218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79622024-01-24T19:08:53.717Z{"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346230731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79632024-01-24T19:08:53.717Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.34625355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79642024-01-24T19:08:53.718Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346295942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79652024-01-24T19:08:53.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346319244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79662024-01-24T19:08:53.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346362713Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79672024-01-24T19:08:53.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346398673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79682024-01-24T19:08:53.719Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346447684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79692024-01-24T19:08:53.719Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346471846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79702024-01-24T19:08:53.719Z{"msg":"b04f7443-49ae-4ffb-b245-8ed384bf6cd0 is now active with session: eb93ce3b-2e90-4908-96a4-141ea8b87c9e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346493888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79712024-01-24T19:08:53.720Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346534908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425,"session_id":"eb93ce3b-2e90-4908-96a4-141ea8b87c9e"}
79722024-01-24T19:08:53.720Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:52.346558431Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1425}
79732024-01-24T19:08:53.720ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79742024-01-24T19:08:53.720ZLoading write count information from file "/tmp/test_repair_verify.data"
79752024-01-24T19:08:53.720ZRun Repair workload
79762024-01-24T19:08:53.721Z01/30 Write block 386 len 36864 data: 5 6 5 5 4 6 5 8 8
79772024-01-24T19:08:53.722Z02/30 Read block 481 len 40960
79782024-01-24T19:08:53.730Z03/30 Write block 356 len 28672 data: 3 2 5 6 7 8 7
79792024-01-24T19:08:53.730Z04/30 Write block 255 len 8192 data: 6 6
79802024-01-24T19:08:53.730Z05/30 Read block 449 len 32768
79812024-01-24T19:08:53.730Z06/30 Read block 111 len 24576
79822024-01-24T19:08:53.730Z07/30 Read block 408 len 20480
79832024-01-24T19:08:53.730Z08/30 Write block 168 len 20480 data: 9 8 7 9 8
79842024-01-24T19:08:53.730Z09/30 Flush
79852024-01-24T19:08:53.731Z10/30 Write block 547 len 4096 data: 8
79862024-01-24T19:08:53.731Z11/30 Write block 307 len 4096 data: 4
79872024-01-24T19:08:53.731Z12/30 Flush
79882024-01-24T19:08:53.731Z13/30 Write block 335 len 36864 data: 7 7 9 9 9 8 6 6 7
79892024-01-24T19:08:53.731Z14/30 Flush
79902024-01-24T19:08:53.731Z15/30 Write block 464 len 16384 data: 4 6 5 7
79912024-01-24T19:08:53.731Z16/30 Write block 365 len 24576 data: 8 8 8 8 8 9
79922024-01-24T19:08:53.731Z17/30 Read block 404 len 12288
79932024-01-24T19:08:53.731Z18/30 Read block 256 len 20480
79942024-01-24T19:08:53.731Z19/30 Read block 107 len 16384
79952024-01-24T19:08:53.731Z20/30 Read block 361 len 12288
79962024-01-24T19:08:53.731Z21/30 Write block 262 len 16384 data: 4 4 7 7
79972024-01-24T19:08:53.731Z22/30 Write block 466 len 12288 data: 6 8 7
79982024-01-24T19:08:53.731Z23/30 Write block 256 len 16384 data: 7 5 6 6
79992024-01-24T19:08:53.731Z24/30 Write block 104 len 8192 data: 8 7
80002024-01-24T19:08:53.732Z25/30 Read block 569 len 4096
80012024-01-24T19:08:53.732Z26/30 Write block 156 len 24576 data: 5 6 5 4 4 8
80022024-01-24T19:08:53.732Z27/30 Write block 267 len 16384 data: 7 7 7 8
80032024-01-24T19:08:53.733Z28/30 Read block 524 len 32768
80042024-01-24T19:08:53.733Z29/30 Write block 220 len 16384 data: 4 5 7 4
80052024-01-24T19:08:54.146Z30/30 Write block 353 len 20480 data: 2 2 2 4 3
80062024-01-24T19:08:54.146Z----------------------------------------------------------------
80072024-01-24T19:08:54.146Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80082024-01-24T19:08:54.146ZStates: Active Active Active
80092024-01-24T19:08:54.146ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80102024-01-24T19:08:54.146Z 1 Acked 1000 Write 9 Done Done Done false
80112024-01-24T19:08:54.146Z 2 Acked 1001 Read 10 Done Done Done false
80122024-01-24T19:08:54.147Z 3 Acked 1002 Write 7 Done Done Done false
80132024-01-24T19:08:54.147Z 4 Acked 1003 Write 2 Sent Done Done false
80142024-01-24T19:08:54.147Z 5 Acked 1004 Read 8 Sent Done Done false
80152024-01-24T19:08:54.147Z 6 Acked 1005 Read 6 Sent Done Done false
80162024-01-24T19:08:54.147Z 7 Acked 1006 Read 5 Sent Done Done false
80172024-01-24T19:08:54.147Z 8 Acked 1007 Write 5 Sent Done Done false
80182024-01-24T19:08:54.147Z 9 Acked 1008 Flush 0 Sent Done Done false
80192024-01-24T19:08:54.147Z 10 Acked 1009 Write 1 Sent Done Done false
80202024-01-24T19:08:54.147Z 11 Acked 1010 Write 1 Sent Done Done false
80212024-01-24T19:08:54.147Z 12 Acked 1011 Flush 0 Sent Done Done false
80222024-01-24T19:08:54.147Z 13 Acked 1012 Write 9 Sent Done Done false
80232024-01-24T19:08:54.147Z 14 Acked 1013 Flush 0 Sent Done Done false
80242024-01-24T19:08:54.148Z 15 Acked 1014 Write 4 Sent Done Done false
80252024-01-24T19:08:54.148Z 16 Acked 1015 Write 6 Sent Done Done false
80262024-01-24T19:08:54.148Z 17 Acked 1016 Read 3 Sent Done Done false
80272024-01-24T19:08:54.149Z 18 Acked 1017 Read 5 Sent Done Done false
80282024-01-24T19:08:54.149Z 19 Acked 1018 Read 4 Sent Done Done false
80292024-01-24T19:08:54.149Z 20 Acked 1019 Read 3 Sent Done Done false
80302024-01-24T19:08:54.149Z 21 Acked 1020 Write 4 Sent Done Done false
80312024-01-24T19:08:54.149Z 22 Acked 1021 Write 3 Sent Done Done false
80322024-01-24T19:08:54.149Z 23 Acked 1022 Write 4 Sent Done Done false
80332024-01-24T19:08:54.149Z 24 Acked 1023 Write 2 Sent Done Done false
80342024-01-24T19:08:54.149Z 25 Acked 1024 Read 1 Sent Done Done false
80352024-01-24T19:08:54.149Z 26 Acked 1025 Write 6 Sent Done Done false
80362024-01-24T19:08:54.149Z 27 Acked 1026 Write 4 Sent Done Done false
80372024-01-24T19:08:54.149Z 28 Acked 1027 Read 8 Sent Done Done false
80382024-01-24T19:08:54.150Z 29 Acked 1028 Write 4 Sent Done Done false
80392024-01-24T19:08:54.150Z 30 Acked 1029 Write 5 Sent Sent Sent false
80402024-01-24T19:08:54.150Z STATES DS:0 DS:1 DS:2 TOTAL
80412024-01-24T19:08:54.150Z New 0 0 0 0
80422024-01-24T19:08:54.150Z Sent 27 1 1 29
80432024-01-24T19:08:54.150Z Done 3 29 29 61
80442024-01-24T19:08:54.150Z Skipped 0 0 0 0
80452024-01-24T19:08:54.150Z Error 0 0 0 0
80462024-01-24T19:08:54.150ZLast Flush: 0 1013 1013
80472024-01-24T19:08:54.150ZDownstairs last five completed:
80482024-01-24T19:08:54.150ZUpstairs last five completed: 30 29 28 27 26
80492024-01-24T19:08:54.151ZWrote out file "/tmp/test_repair_verify.data"
80502024-01-24T19:08:54.163Z
80512024-01-24T19:08:54.163ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80522024-01-24T19:08:54.163ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80532024-01-24T19:08:54.163Z 0 000-019 70 70 70 116 116 116 F F F
80542024-01-24T19:08:54.164Z 1 020-039 60 60 60 94 94 94 F F F
80552024-01-24T19:08:54.164Z 2 040-059 68 68 68 113 113 113 F F F
80562024-01-24T19:08:54.164Z 3 060-079 71 71 71 117 117 117 F F F
80572024-01-24T19:08:54.164Z 4 080-099 72 72 72 118 118 118 F F F
80582024-01-24T19:08:54.164Z 5 100-119 72 72 72 118 118 118 F T T <---
80592024-01-24T19:08:54.164Z 6 120-139 63 63 63 104 104 104 F F F
80602024-01-24T19:08:54.164Z 7 140-159 66 66 66 112 112 112 F T T <---
80612024-01-24T19:08:54.164Z 8 160-179 68 74 74 114 119 119 F T T <---
80622024-01-24T19:08:54.164Z 9 180-199 72 72 72 118 118 118 F F F
80632024-01-24T19:08:54.165Z 10 200-219 68 68 68 113 113 113 F F F
80642024-01-24T19:08:54.165Z 11 220-239 70 70 70 116 116 116 F T T <---
80652024-01-24T19:08:54.165Z 12 240-259 68 74 74 113 119 119 F T T <---
80662024-01-24T19:08:54.165Z 13 260-279 72 72 72 118 118 118 F T T <---
80672024-01-24T19:08:54.169Z 14 280-299 72 72 72 118 118 118 F F F
80682024-01-24T19:08:54.169Z 15 300-319 72 74 74 118 120 120 F F F <---
80692024-01-24T19:08:54.169Z 16 320-339 72 74 74 118 121 121 F F F <---
80702024-01-24T19:08:54.170Z 17 340-359 66 74 74 112 121 121 T T T <---
80712024-01-24T19:08:54.170Z 18 360-379 66 74 74 112 119 119 T T T <---
80722024-01-24T19:08:54.170Z 19 380-399 64 74 74 110 119 119 T F F <---
80732024-01-24T19:08:54.172Z 20 400-419 64 64 64 107 107 107 F F F
80742024-01-24T19:08:54.172Z 21 420-439 68 68 68 113 113 113 F F F
80752024-01-24T19:08:54.173Z 22 440-459 71 71 71 117 117 117 F F F
80762024-01-24T19:08:54.173Z 23 460-479 69 69 69 115 115 115 F T T <---
80772024-01-24T19:08:54.173Z 24 480-499 71 71 71 117 117 117 F F F
80782024-01-24T19:08:54.173Z 25 500-519 72 72 72 118 118 118 F F F
80792024-01-24T19:08:54.173Z 26 520-539 72 72 72 118 118 118 F F F
80802024-01-24T19:08:54.174Z 27 540-559 66 74 74 112 120 120 F F F <---
80812024-01-24T19:08:54.174Z 28 560-579 72 72 72 118 118 118 F F F
80822024-01-24T19:08:54.174Z 29 580-599 72 72 72 118 118 118 F F F
80832024-01-24T19:08:54.174ZMax gen: 74, Max flush: 121
80842024-01-24T19:08:54.174ZError: Difference in extent metadata found!
80852024-01-24T19:08:54.174ZA Difference in extent metadata is expected here
80862024-01-24T19:08:54.175ZOn loop 37
80872024-01-24T19:08:54.175Z
80882024-01-24T19:08:54.175ZVerifying data now
80892024-01-24T19:08:59.985ZNo change after verify
80902024-01-24T19:08:59.985ZLoop: 37 Downstairs dump after verify (and repair):
80912024-01-24T19:08:59.995ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80922024-01-24T19:08:59.995Z 0 000-019 70 70 70 116 116 116 F F F
80932024-01-24T19:08:59.995Z 1 020-039 60 60 60 94 94 94 F F F
80942024-01-24T19:08:59.996Z 2 040-059 68 68 68 113 113 113 F F F
80952024-01-24T19:09:00.012Z 3 060-079 71 71 71 117 117 117 F F F
80962024-01-24T19:09:00.012Z 4 080-099 72 72 72 118 118 118 F F F
80972024-01-24T19:09:00.012Z 5 100-119 75 75 75 122 122 122 F F F
80982024-01-24T19:09:00.012Z 6 120-139 63 63 63 104 104 104 F F F
80992024-01-24T19:09:00.012Z 7 140-159 75 75 75 122 122 122 F F F
81002024-01-24T19:09:00.013Z 8 160-179 75 75 75 122 122 122 F F F
81012024-01-24T19:09:00.013Z 9 180-199 72 72 72 118 118 118 F F F
81022024-01-24T19:09:00.013Z 10 200-219 68 68 68 113 113 113 F F F
81032024-01-24T19:09:00.013Z 11 220-239 75 75 75 122 122 122 F F F
81042024-01-24T19:09:00.013Z 12 240-259 75 75 75 122 122 122 F F F
81052024-01-24T19:09:00.013Z 13 260-279 75 75 75 122 122 122 F F F
81062024-01-24T19:09:00.014Z 14 280-299 72 72 72 118 118 118 F F F
81072024-01-24T19:09:00.014Z 15 300-319 74 74 74 120 120 120 F F F
81082024-01-24T19:09:00.014Z 16 320-339 74 74 74 121 121 121 F F F
81092024-01-24T19:09:00.014Z 17 340-359 75 75 75 122 122 122 F F F
81102024-01-24T19:09:00.014Z 18 360-379 75 75 75 122 122 122 F F F
81112024-01-24T19:09:00.014Z 19 380-399 74 74 74 119 119 119 F F F
81122024-01-24T19:09:00.014Z 20 400-419 64 64 64 107 107 107 F F F
81132024-01-24T19:09:00.014Z 21 420-439 68 68 68 113 113 113 F F F
81142024-01-24T19:09:00.014Z 22 440-459 71 71 71 117 117 117 F F F
81152024-01-24T19:09:00.015Z 23 460-479 75 75 75 122 122 122 F F F
81162024-01-24T19:09:00.015Z 24 480-499 71 71 71 117 117 117 F F F
81172024-01-24T19:09:00.015Z 25 500-519 72 72 72 118 118 118 F F F
81182024-01-24T19:09:00.015Z 26 520-539 72 72 72 118 118 118 F F F
81192024-01-24T19:09:00.015Z 27 540-559 74 74 74 120 120 120 F F F
81202024-01-24T19:09:00.015Z 28 560-579 72 72 72 118 118 118 F F F
81212024-01-24T19:09:00.015Z 29 580-599 72 72 72 118 118 118 F F F
81222024-01-24T19:09:00.016ZMax gen: 75, Max flush: 122
81232024-01-24T19:09:00.016Z
81242024-01-24T19:09:00.016ZBegin loop 38
81252024-01-24T19:09:00.016ZDownstairs to restart: 2
81262024-01-24T19:09:00.026Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.662564702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433}
81272024-01-24T19:09:00.027Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663090552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433}
81282024-01-24T19:09:00.027Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663119725Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433}
81292024-01-24T19:09:00.027Z{"msg":"Crucible d6292430-bb0c-4c0d-be9a-4a879a768bc0 has session id: be561f47-00f6-474b-af9a-8ca4de8852a3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663173068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81302024-01-24T19:09:00.027Z{"msg":"Upstairs opts: Upstairs UUID: d6292430-bb0c-4c0d-be9a-4a879a768bc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663211285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81312024-01-24T19:09:00.027Z{"msg":"Crucible stats registered with UUID: d6292430-bb0c-4c0d-be9a-4a879a768bc0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663242193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81322024-01-24T19:09:00.028ZCrucible runtime is spawned
81332024-01-24T19:09:00.028Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663519619Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433}
81342024-01-24T19:09:00.029Z{"msg":"d6292430-bb0c-4c0d-be9a-4a879a768bc0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663830077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81352024-01-24T19:09:00.029Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.663891992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81362024-01-24T19:09:00.029Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664032841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81372024-01-24T19:09:00.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664112809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81382024-01-24T19:09:00.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664168486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81392024-01-24T19:09:00.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664302161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81402024-01-24T19:09:00.031Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664570704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81412024-01-24T19:09:00.031Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664753809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81422024-01-24T19:09:00.031Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664801917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81432024-01-24T19:09:00.031Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664845132Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81442024-01-24T19:09:00.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.66488558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81452024-01-24T19:09:00.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664956824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81462024-01-24T19:09:00.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.664985512Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81472024-01-24T19:09:00.033Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.665911109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81482024-01-24T19:09:00.034Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.665989777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81492024-01-24T19:09:00.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.666035631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81502024-01-24T19:09:00.034Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.66608155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81512024-01-24T19:09:00.035Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.666226128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81522024-01-24T19:09:00.035Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.666311224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81532024-01-24T19:09:00.035Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.667008619Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81542024-01-24T19:09:00.037Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.667071263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81552024-01-24T19:09:00.037Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.667195867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81562024-01-24T19:09:00.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.667871062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81572024-01-24T19:09:00.037Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.667956704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81582024-01-24T19:09:00.037Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668004135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81592024-01-24T19:09:00.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668043295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81602024-01-24T19:09:00.038Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668110698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81612024-01-24T19:09:00.038Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668153745Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81622024-01-24T19:09:00.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668225878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81632024-01-24T19:09:00.040Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.66828091Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81642024-01-24T19:09:00.049Z{"msg":"[0]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 122, 104, 122, 122, 118, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668308848Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81652024-01-24T19:09:00.049Z{"msg":"[0]R generation[0..12]: [70, 60, 68, 71, 72, 75, 63, 75, 75, 72, 68, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668353405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81662024-01-24T19:09:00.049Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668380984Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81672024-01-24T19:09:00.049Z{"msg":"[1]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 122, 104, 122, 122, 118, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668425927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81682024-01-24T19:09:00.050Z{"msg":"[1]R generation[0..12]: [70, 60, 68, 71, 72, 75, 63, 75, 75, 72, 68, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668452129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81692024-01-24T19:09:00.050Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.66850012Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81702024-01-24T19:09:00.050Z{"msg":"[2]R flush_numbers[0..12]: [116, 94, 113, 117, 118, 122, 104, 122, 122, 118, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668527125Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
81712024-01-24T19:09:00.050Zbe561f47-00f6-474b-af9a-8ca4de8852a3"}
81722024-01-24T19:09:00.051Z{"msg":"[2]R generation[0..12]: [70, 60, 68, 71, 72, 75, 63, 75, 75, 72, 68, 75]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668620433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81732024-01-24T19:09:00.051Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668672888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81742024-01-24T19:09:00.056Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668734691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81752024-01-24T19:09:00.056Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668760515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81762024-01-24T19:09:00.056Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668781847Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81772024-01-24T19:09:00.056Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668803161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81782024-01-24T19:09:00.056Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668833797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81792024-01-24T19:09:00.057Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.66885569Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81802024-01-24T19:09:00.057Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668877655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81812024-01-24T19:09:00.057Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668899135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81822024-01-24T19:09:00.057Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668937497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81832024-01-24T19:09:00.057Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.668989886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81842024-01-24T19:09:00.059Z{"msg":"d6292430-bb0c-4c0d-be9a-4a879a768bc0 is now active with session: be561f47-00f6-474b-af9a-8ca4de8852a3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.669017128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81852024-01-24T19:09:00.059Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.669037298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433,"session_id":"be561f47-00f6-474b-af9a-8ca4de8852a3"}
81862024-01-24T19:09:00.059Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:08:58.669057829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1433}
81872024-01-24T19:09:00.059ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
81882024-01-24T19:09:00.060ZLoading write count information from file "/tmp/test_repair_verify.data"
81892024-01-24T19:09:00.060ZRun Repair workload
81902024-01-24T19:09:00.060Z01/30 Flush
81912024-01-24T19:09:00.060Z02/30 Write block 28 len 36864 data: 9 8 8 9 8 9 9 4 4
81922024-01-24T19:09:00.060Z03/30 Read block 325 len 4096
81932024-01-24T19:09:00.060Z04/30 Read block 237 len 28672
81942024-01-24T19:09:00.061Z05/30 Write block 510 len 4096 data: 9
81952024-01-24T19:09:00.061Z06/30 Flush
81962024-01-24T19:09:00.061Z07/30 Write block 124 len 16384 data: 8 7 7 6
81972024-01-24T19:09:00.061Z08/30 Read block 237 len 36864
81982024-01-24T19:09:00.061Z09/30 Read block 488 len 28672
81992024-01-24T19:09:00.061Z10/30 Read block 473 len 40960
82002024-01-24T19:09:00.069Z11/30 Write block 233 len 32768 data: 7 7 5 3 3 3 4 5
82012024-01-24T19:09:00.069Z12/30 Write block 280 len 36864 data: 9 9 9 8 5 3 3 5 7
82022024-01-24T19:09:00.069Z13/30 Write block 344 len 20480 data: 7 7 4 3 3
82032024-01-24T19:09:00.069Z14/30 Read block 49 len 24576
82042024-01-24T19:09:00.070Z15/30 Write block 419 len 16384 data: 8 6 6 8
82052024-01-24T19:09:00.070Z16/30 Write block 240 len 16384 data: 6 6 7 9
82062024-01-24T19:09:00.070Z17/30 Flush
82072024-01-24T19:09:00.070Z18/30 Write block 490 len 40960 data: 4 3 5 5 6 6 8 8 8 7
82082024-01-24T19:09:00.070Z19/30 Read block 269 len 4096
82092024-01-24T19:09:00.070Z20/30 Read block 167 len 36864
82102024-01-24T19:09:00.070Z21/30 Write block 431 len 40960 data: 8 9 9 9 7 6 7 5 5 4
82112024-01-24T19:09:00.070Z22/30 Read block 213 len 32768
82122024-01-24T19:09:00.070Z23/30 Write block 64 len 12288 data: 6 5 2
82132024-01-24T19:09:00.070Z24/30 Write block 565 len 36864 data: 5 5 5 5 5 3 5 6 5
82142024-01-24T19:09:00.070Z25/30 Write block 491 len 36864 data: 4 6 6 7 7 9 9 9 8
82152024-01-24T19:09:00.070Z26/30 Read block 96 len 40960
82162024-01-24T19:09:00.070Z27/30 Write block 272 len 40960 data: 9 10 7 6 5 8 9 11 10 10
82172024-01-24T19:09:00.070Z28/30 Read block 141 len 16384
82182024-01-24T19:09:00.070Z29/30 Write block 163 len 16384 data: 9 8 8 6
82192024-01-24T19:09:00.070Z30/30 Write block 375 len 28672 data: 9 8 8 5 5 7 5
82202024-01-24T19:09:00.070Z----------------------------------------------------------------
82212024-01-24T19:09:00.071Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:29
82222024-01-24T19:09:00.071ZStates: Active Active Active
82232024-01-24T19:09:00.071ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82242024-01-24T19:09:00.071Z 2 Acked 1001 Write 9 Done Done Done false
82252024-01-24T19:09:00.071Z 3 Acked 1002 Read 1 Done Done Done false
82262024-01-24T19:09:00.071Z 4 Acked 1003 Read 7 Done Done Sent false
82272024-01-24T19:09:00.071Z 5 Acked 1004 Write 1 Done Done Sent false
82282024-01-24T19:09:00.072Z 6 Acked 1005 Flush 0 Done Done Sent false
82292024-01-24T19:09:00.072Z 7 Acked 1006 Write 4 Done Done Sent false
82302024-01-24T19:09:00.072Z 8 Acked 1007 Read 9 Done Done Sent false
82312024-01-24T19:09:00.072Z 9 Acked 1008 Read 7 Done Done Sent false
82322024-01-24T19:09:00.072Z 10 Acked 1009 Read 10 Done Done Sent false
82332024-01-24T19:09:00.072Z 11 Acked 1010 Write 8 Done Done Sent false
82342024-01-24T19:09:00.072Z 12 Acked 1011 Write 9 Done Done Sent false
82352024-01-24T19:09:00.072Z 13 Acked 1012 Write 5 Done Done Sent false
82362024-01-24T19:09:00.072Z 14 Acked 1013 Read 6 Done Done Sent false
82372024-01-24T19:09:00.072Z 15 Acked 1014 Write 4 Done Done Sent false
82382024-01-24T19:09:00.072Z 16 Acked 1015 Write 4 Done Done Sent false
82392024-01-24T19:09:00.072Z 17 Acked 1016 Flush 0 Done Done Sent false
82402024-01-24T19:09:00.072Z 18 Acked 1017 Write 10 Done Done Sent false
82412024-01-24T19:09:00.072Z 19 Acked 1018 Read 1 Done Done Sent false
82422024-01-24T19:09:00.072Z 20 Acked 1019 Read 9 Done Done Sent false
82432024-01-24T19:09:00.072Z 21 Acked 1020 Write 10 Done Done Sent false
82442024-01-24T19:09:00.072Z 22 Acked 1021 Read 8 Done Done Sent false
82452024-01-24T19:09:00.072Z 23 Acked 1022 Write 3 Done Done Sent false
82462024-01-24T19:09:00.072Z 24 Acked 1023 Write 9 Done Done Sent false
82472024-01-24T19:09:00.072Z 25 Acked 1024 Write 9 Done Done Sent false
82482024-01-24T19:09:00.072Z 26 Acked 1025 Read 10 Done Done Sent false
82492024-01-24T19:09:00.072Z 27 Acked 1026 Write 10 Done Done Sent false
82502024-01-24T19:09:00.072Z 28 Acked 1027 Read 4 Done Done Sent false
82512024-01-24T19:09:00.073Z 29 Acked 1028 Write 4 Done Done Sent false
82522024-01-24T19:09:00.074Z 30 Acked 1029 Write 7 Sent Sent Sent false
82532024-01-24T19:09:00.074Z STATES DS:0 DS:1 DS:2 TOTAL
82542024-01-24T19:09:00.074Z New 0 0 0 0
82552024-01-24T19:09:00.074Z Sent 1 1 27 29
82562024-01-24T19:09:00.074Z Done 28 28 2 58
82572024-01-24T19:09:00.074Z Skipped 0 0 0 0
82582024-01-24T19:09:00.074Z Error 0 0 0 0
82592024-01-24T19:09:00.074ZLast Flush: 1016 1016 1000
82602024-01-24T19:09:00.074ZDownstairs last five completed: 1000
82612024-01-24T19:09:00.074ZUpstairs last five completed: 30 29 28 27 26
82622024-01-24T19:09:00.074ZWrote out file "/tmp/test_repair_verify.data"
82632024-01-24T19:09:00.074Z
82642024-01-24T19:09:00.077ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
82652024-01-24T19:09:00.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82662024-01-24T19:09:00.096Z 0 000-019 70 70 70 116 116 116 F F F
82672024-01-24T19:09:00.099Z 1 020-039 76 76 60 124 124 94 F F T <---
82682024-01-24T19:09:00.100Z 2 040-059 68 68 68 113 113 113 F F F
82692024-01-24T19:09:00.100Z 3 060-079 71 71 71 117 117 117 T T F <---
82702024-01-24T19:09:00.100Z 4 080-099 72 72 72 118 118 118 F F F
82712024-01-24T19:09:00.100Z 5 100-119 75 75 75 122 122 122 F F F
82722024-01-24T19:09:00.100Z 6 120-139 76 76 63 125 125 104 F F F <---
82732024-01-24T19:09:00.100Z 7 140-159 75 75 75 122 122 122 F F F
82742024-01-24T19:09:00.100Z 8 160-179 75 75 75 122 122 122 T T F <---
82752024-01-24T19:09:00.100Z 9 180-199 72 72 72 118 118 118 F F F
82762024-01-24T19:09:00.100Z 10 200-219 68 68 68 113 113 113 F F F
82772024-01-24T19:09:00.100Z 11 220-239 76 76 75 125 125 122 F F F <---
82782024-01-24T19:09:00.100Z 12 240-259 76 76 75 125 125 122 F F F <---
82792024-01-24T19:09:00.100Z 13 260-279 75 75 75 122 122 122 T T F <---
82802024-01-24T19:09:00.100Z 14 280-299 76 76 72 125 125 118 T T F <---
82812024-01-24T19:09:00.100Z 15 300-319 74 74 74 120 120 120 F F F
82822024-01-24T19:09:00.100Z 16 320-339 74 74 74 121 121 121 F F F
82832024-01-24T19:09:00.100Z 17 340-359 76 76 75 125 125 122 F F F <---
82842024-01-24T19:09:00.100Z 18 360-379 75 75 75 122 122 122 T T F <---
82852024-01-24T19:09:00.100Z 19 380-399 74 74 74 119 119 119 T T F <---
82862024-01-24T19:09:00.100Z 20 400-419 76 76 64 125 125 107 F F F <---
82872024-01-24T19:09:00.100Z 21 420-439 76 76 68 125 125 113 T T F <---
82882024-01-24T19:09:00.100Z 22 440-459 71 71 71 117 117 117 T T F <---
82892024-01-24T19:09:00.100Z 23 460-479 75 75 75 122 122 122 F F F
82902024-01-24T19:09:00.101Z 24 480-499 71 71 71 117 117 117 T T F <---
82912024-01-24T19:09:00.101Z 25 500-519 76 76 72 124 124 118 F F F <---
82922024-01-24T19:09:00.101Z 26 520-539 72 72 72 118 118 118 F F F
82932024-01-24T19:09:00.101Z 27 540-559 74 74 74 120 120 120 F F F
82942024-01-24T19:09:00.101Z 28 560-579 72 72 72 118 118 118 T T F <---
82952024-01-24T19:09:00.101Z 29 580-599 72 72 72 118 118 118 F F F
82962024-01-24T19:09:00.101ZMax gen: 76, Max flush: 125
82972024-01-24T19:09:00.101ZError: Difference in extent metadata found!
82982024-01-24T19:09:00.104ZA Difference in extent metadata is expected here
82992024-01-24T19:09:00.104ZOn loop 38
83002024-01-24T19:09:00.104Z
83012024-01-24T19:09:00.104ZVerifying data now
83022024-01-24T19:09:05.657ZNo change after verify
83032024-01-24T19:09:05.658ZLoop: 38 Downstairs dump after verify (and repair):
83042024-01-24T19:09:05.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83052024-01-24T19:09:05.665Z 0 000-019 70 70 70 116 116 116 F F F
83062024-01-24T19:09:05.665Z 1 020-039 76 76 76 124 124 124 F F F
83072024-01-24T19:09:05.666Z 2 040-059 68 68 68 113 113 113 F F F
83082024-01-24T19:09:05.666Z 3 060-079 77 77 77 126 126 126 F F F
83092024-01-24T19:09:05.666Z 4 080-099 72 72 72 118 118 118 F F F
83102024-01-24T19:09:05.666Z 5 100-119 75 75 75 122 122 122 F F F
83112024-01-24T19:09:05.666Z 6 120-139 76 76 76 125 125 125 F F F
83122024-01-24T19:09:05.666Z 7 140-159 75 75 75 122 122 122 F F F
83132024-01-24T19:09:05.666Z 8 160-179 77 77 77 126 126 126 F F F
83142024-01-24T19:09:05.666Z 9 180-199 72 72 72 118 118 118 F F F
83152024-01-24T19:09:05.670Z 10 200-219 68 68 68 113 113 113 F F F
83162024-01-24T19:09:05.670Z 11 220-239 76 76 76 125 125 125 F F F
83172024-01-24T19:09:05.670Z 12 240-259 76 76 76 125 125 125 F F F
83182024-01-24T19:09:05.671Z 13 260-279 77 77 77 126 126 126 F F F
83192024-01-24T19:09:05.671Z 14 280-299 77 77 77 126 126 126 F F F
83202024-01-24T19:09:05.671Z 15 300-319 74 74 74 120 120 120 F F F
83212024-01-24T19:09:05.671Z 16 320-339 74 74 74 121 121 121 F F F
83222024-01-24T19:09:05.671Z 17 340-359 76 76 76 125 125 125 F F F
83232024-01-24T19:09:05.671Z 18 360-379 77 77 77 126 126 126 F F F
83242024-01-24T19:09:05.671Z 19 380-399 77 77 77 126 126 126 F F F
83252024-01-24T19:09:05.671Z 20 400-419 76 76 76 125 125 125 F F F
83262024-01-24T19:09:05.671Z 21 420-439 77 77 77 126 126 126 F F F
83272024-01-24T19:09:05.672Z 22 440-459 77 77 77 126 126 126 F F F
83282024-01-24T19:09:05.672Z 23 460-479 75 75 75 122 122 122 F F F
83292024-01-24T19:09:05.672Z 24 480-499 77 77 77 126 126 126 F F F
83302024-01-24T19:09:05.672Z 25 500-519 76 76 76 124 124 124 F F F
83312024-01-24T19:09:05.672Z 26 520-539 72 72 72 118 118 118 F F F
83322024-01-24T19:09:05.672Z 27 540-559 74 74 74 120 120 120 F F F
83332024-01-24T19:09:05.672Z 28 560-579 77 77 77 126 126 126 F F F
83342024-01-24T19:09:05.672Z 29 580-599 72 72 72 118 118 118 F F F
83352024-01-24T19:09:05.672ZMax gen: 77, Max flush: 126
83362024-01-24T19:09:05.672Z
83372024-01-24T19:09:05.672ZBegin loop 39
83382024-01-24T19:09:05.672ZDownstairs to restart: 1
83392024-01-24T19:09:05.672Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.292805571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441}
83402024-01-24T19:09:05.673Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293280994Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441}
83412024-01-24T19:09:05.673Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293311388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441}
83422024-01-24T19:09:05.673Z{"msg":"Crucible bf42e7c5-b4ce-42ec-9f53-f337a2270706 has session id: 8ac14380-31fe-4a56-9909-ad9fd134a9d3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293356278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83432024-01-24T19:09:05.673Z{"msg":"Upstairs opts: Upstairs UUID: bf42e7c5-b4ce-42ec-9f53-f337a2270706, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293395225Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83442024-01-24T19:09:05.673Z{"msg":"Crucible stats registered with UUID: bf42e7c5-b4ce-42ec-9f53-f337a2270706","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29347084Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83452024-01-24T19:09:05.673ZCrucible runtime is spawned
83462024-01-24T19:09:05.673Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293675292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441}
83472024-01-24T19:09:05.673Z{"msg":"bf42e7c5-b4ce-42ec-9f53-f337a2270706 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.293938145Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83482024-01-24T19:09:05.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294044256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83492024-01-24T19:09:05.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294127268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83502024-01-24T19:09:05.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29417005Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83512024-01-24T19:09:05.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294250836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83522024-01-24T19:09:05.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294439129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83532024-01-24T19:09:05.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294574413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83542024-01-24T19:09:05.676Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29469204Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83552024-01-24T19:09:05.676Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294736358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83562024-01-24T19:09:05.676Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294776924Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83572024-01-24T19:09:05.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294816321Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83582024-01-24T19:09:05.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29485792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83592024-01-24T19:09:05.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.294933283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83602024-01-24T19:09:05.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296149529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83612024-01-24T19:09:05.678Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296336834Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83622024-01-24T19:09:05.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296425502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83632024-01-24T19:09:05.678Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296471835Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83642024-01-24T19:09:05.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296609698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83652024-01-24T19:09:05.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.296673892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83662024-01-24T19:09:05.679Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.297387316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83672024-01-24T19:09:05.679Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.297478242Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83682024-01-24T19:09:05.679Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.297527631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83692024-01-24T19:09:05.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298459317Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83702024-01-24T19:09:05.679Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298537565Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83712024-01-24T19:09:05.679Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298584003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83722024-01-24T19:09:05.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298635638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83732024-01-24T19:09:05.679Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298668356Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83742024-01-24T19:09:05.679Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298715523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83752024-01-24T19:09:05.679Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298740401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83762024-01-24T19:09:05.679Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298784667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83772024-01-24T19:09:05.680Z{"msg":"[0]R flush_numbers[0..12]: [116, 124, 113, 126, 118, 122, 125, 122, 126, 118, 113, 125]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298811558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83782024-01-24T19:09:05.680Z{"msg":"[0]R generation[0..12]: [70, 76, 68, 77, 72, 75, 76, 75, 77, 72, 68, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298855534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83792024-01-24T19:09:05.680Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.298882921Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83802024-01-24T19:09:05.680Z{"msg":"[1]R flush_numbers[0..12]: [116, 124, 113, 126, 118, 122, 125, 122, 126, 118, 113, 125]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29892682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83812024-01-24T19:09:05.686Z{"msg":"[1]R generation[0..12]: [70, 76, 68, 77, 72, 75, 76, 75, 77, 72, 68, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29895333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83822024-01-24T19:09:05.686Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299029341Z","hostname":"Wait for a query_work_queue command to finish before sending IO
83832024-01-24T19:09:05.686Zip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83842024-01-24T19:09:05.686Z{"msg":"[2]R flush_numbers[0..12]: [116, 124, 113, 126, 118, 122, 125, 122, 126, 118, 113, 125]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299078051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83852024-01-24T19:09:05.686Z{"msg":"[2]R generation[0..12]: [70, 76, 68, 77, 72, 75, 76, 75, 77, 72, 68, 76]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299122397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83862024-01-24T19:09:05.686Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299163766Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83872024-01-24T19:09:05.686Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.29922017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83882024-01-24T19:09:05.686Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299264522Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83892024-01-24T19:09:05.686Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299328251Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83902024-01-24T19:09:05.687Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299372071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83912024-01-24T19:09:05.688Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299437879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83922024-01-24T19:09:05.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299476977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83932024-01-24T19:09:05.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299545435Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83942024-01-24T19:09:05.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299588377Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"client":"2","":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
83952024-01-24T19:09:05.689Zdownstairs","session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83962024-01-24T19:09:05.689Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299688895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83972024-01-24T19:09:05.689Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299747035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83982024-01-24T19:09:05.689Z{"msg":"bf42e7c5-b4ce-42ec-9f53-f337a2270706 is now active with session: 8ac14380-31fe-4a56-9909-ad9fd134a9d3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299790327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
83992024-01-24T19:09:05.689Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299843643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441,"session_id":"8ac14380-31fe-4a56-9909-ad9fd134a9d3"}
84002024-01-24T19:09:05.689Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:04.299883508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1441}
84012024-01-24T19:09:05.689ZLoading write count information from file "/tmp/test_repair_verify.data"
84022024-01-24T19:09:05.689ZRun Repair workload
84032024-01-24T19:09:05.690Z01/30 Write block 455 len 16384 data: 6 8 8 8
84042024-01-24T19:09:05.799Z02/30 Read block 312 len 28672
84052024-01-24T19:09:05.799Z03/30 Read block 2 len 12288
84062024-01-24T19:09:05.799Z04/30 Write block 236 len 4096 data: 4
84072024-01-24T19:09:05.799Z05/30 Write block 300 len 12288 data: 5 4 4
84082024-01-24T19:09:05.800Z06/30 Write block 259 len 28672 data: 7 6 5 5 5 8 8
84092024-01-24T19:09:05.800Z07/30 Write block 161 len 32768 data: 9 9 10 9 9 7 8 10
84102024-01-24T19:09:05.800Z08/30 Read block 88 len 20480
84112024-01-24T19:09:05.800Z09/30 Write block 110 len 36864 data: 7 6 6 5 3 4 3 3 4
84122024-01-24T19:09:05.800Z10/30 Flush
84132024-01-24T19:09:05.800Z11/30 Write block 45 len 24576 data: 9 7 5 7 7 9
84142024-01-24T19:09:05.801Z12/30 Flush
84152024-01-24T19:09:05.801Z13/30 Write block 282 len 16384 data: 10 9 6 4
84162024-01-24T19:09:05.801Z14/30 Read block 300 len 4096
84172024-01-24T19:09:05.801Z15/30 Read block 331 len 8192
84182024-01-24T19:09:05.801Z16/30 Read block 372 len 40960
84192024-01-24T19:09:05.802Z17/30 Read block 108 len 24576
84202024-01-24T19:09:05.802Z18/30 Write block 201 len 40960 data: 4 4 4 6 6 5 5 5 6 8
84212024-01-24T19:09:05.802Z19/30 Flush
84222024-01-24T19:09:05.802Z20/30 Write block 124 len 8192 data: 9 8
84232024-01-24T19:09:05.802Z21/30 Write block 528 len 24576 data: 10 9 8 6 6 5
84242024-01-24T19:09:05.840Z22/30 Read block 390 len 8192
84252024-01-24T19:09:05.840Z23/30 Read block 356 len 16384
84262024-01-24T19:09:05.840Z24/30 Read block 534 len 8192
84272024-01-24T19:09:05.840Z25/30 Write block 203 len 12288 data: 5 7 7
84282024-01-24T19:09:05.840Z26/30 Write block 242 len 40960 data: 8 10 9 9 11 9 8 9 6 5
84292024-01-24T19:09:05.844Z27/30 Write block 267 len 36864 data: 8 8 8 9 8 10 11 8 7
84302024-01-24T19:09:05.844Z28/30 Write block 216 len 16384 data: 6 6 6 5
84312024-01-24T19:09:05.844Z29/30 Read block 524 len 32768
84322024-01-24T19:09:05.845Z30/30 Read block 224 len 20480
84332024-01-24T19:09:05.845Z----------------------------------------------------------------
84342024-01-24T19:09:05.845Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84352024-01-24T19:09:05.845ZStates: Active Active Active
84362024-01-24T19:09:05.845ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84372024-01-24T19:09:05.845Z 1 Acked 1000 Write 4 Done Done Done false
84382024-01-24T19:09:05.845Z 2 Acked 1001 Read 7 Done Sent Done false
84392024-01-24T19:09:05.845Z 3 Acked 1002 Read 3 Done Sent Done false
84402024-01-24T19:09:05.845Z 4 Acked 1003 Write 1 Done Sent Done false
84412024-01-24T19:09:05.845Z 5 Acked 1004 Write 3 Done Sent Done false
84422024-01-24T19:09:05.846Z 6 Acked 1005 Write 7 Done Sent Done false
84432024-01-24T19:09:05.846Z 7 Acked 1006 Write 8 Done Sent Done false
84442024-01-24T19:09:05.846Z 8 Acked 1007 Read 5 Done Sent Done false
84452024-01-24T19:09:05.846Z 9 Acked 1008 Write 9 Done Sent Done false
84462024-01-24T19:09:05.846Z 10 Acked 1009 Flush 0 Done Sent Done false
84472024-01-24T19:09:05.846Z 11 Acked 1010 Write 6 Done Sent Done false
84482024-01-24T19:09:05.846Z 12 Acked 1011 Flush 0 Done Sent Done false
84492024-01-24T19:09:05.846Z 13 Acked 1012 Write 4 Done Sent Done false
84502024-01-24T19:09:05.846Z 14 Acked 1013 Read 1 Done Sent Done false
84512024-01-24T19:09:05.846Z 15 Acked 1014 Read 2 Done Sent Done false
84522024-01-24T19:09:05.846Z 16 Acked 1015 Read 10 Done Sent Done false
84532024-01-24T19:09:05.846Z 17 Acked 1016 Read 6 Done Sent Done false
84542024-01-24T19:09:05.846Z 18 Acked 1017 Write 10 Done Sent Done false
84552024-01-24T19:09:05.846Z 19 Acked 1018 Flush 0 Done Sent Done false
84562024-01-24T19:09:05.846Z 20 Acked 1019 Write 2 Done Sent Done false
84572024-01-24T19:09:05.846Z 21 Acked 1020 Write 6 Done Sent Done false
84582024-01-24T19:09:05.847Z 22 Acked 1021 Read 2 Done Sent Done false
84592024-01-24T19:09:05.847Z 23 Acked 1022 Read 4 Done Sent Done false
84602024-01-24T19:09:05.847Z 24 Acked 1023 Read 2 Done Sent Done false
84612024-01-24T19:09:05.847Z 25 Acked 1024 Write 3 Done Sent Done false
84622024-01-24T19:09:05.847Z 26 Acked 1025 Write 10 Done Sent Done false
84632024-01-24T19:09:05.847Z 27 Acked 1026 Write 9 Done Sent Done false
84642024-01-24T19:09:05.847Z 28 Acked 1027 Write 4 Done Sent Done false
84652024-01-24T19:09:05.848Z 29 Acked 1028 Read 8 Done Sent Done false
84662024-01-24T19:09:05.848Z 30 Acked 1029 Read 5 Sent Sent Done false
84672024-01-24T19:09:05.848Z STATES DS:0 DS:1 DS:2 TOTAL
84682024-01-24T19:09:05.848Z New 0 0 0 0
84692024-01-24T19:09:05.848Z Sent 1 29 0 30
84702024-01-24T19:09:05.848Z Done 29 1 30 60
84712024-01-24T19:09:05.849Z Skipped 0 0 0 0
84722024-01-24T19:09:05.849Z Error 0 0 0 0
84732024-01-24T19:09:05.849ZLast Flush: 1018 0 1018
84742024-01-24T19:09:05.849ZDownstairs last five completed:
84752024-01-24T19:09:05.850ZUpstairs last five completed: 30 29 28 27 26
84762024-01-24T19:09:05.850ZWrote out file "/tmp/test_repair_verify.data"
84772024-01-24T19:09:05.850Z
84782024-01-24T19:09:05.850ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
84792024-01-24T19:09:05.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84802024-01-24T19:09:05.851Z 0 000-019 70 70 70 116 116 116 F F F
84812024-01-24T19:09:05.851Z 1 020-039 76 76 76 124 124 124 F F F
84822024-01-24T19:09:05.851Z 2 040-059 78 68 78 128 113 128 F F F <---
84832024-01-24T19:09:05.851Z 3 060-079 77 77 77 126 126 126 F F F
84842024-01-24T19:09:05.851Z 4 080-099 72 72 72 118 118 118 F F F
84852024-01-24T19:09:05.851Z 5 100-119 78 75 78 127 122 127 F F F <---
84862024-01-24T19:09:05.851Z 6 120-139 76 76 76 125 125 125 T F T <---
84872024-01-24T19:09:05.851Z 7 140-159 75 75 75 122 122 122 F F F
84882024-01-24T19:09:05.851Z 8 160-179 78 77 78 127 126 127 F F F <---
84892024-01-24T19:09:05.851Z 9 180-199 72 72 72 118 118 118 F F F
84902024-01-24T19:09:05.851Z 10 200-219 78 68 78 129 113 129 T F T <---
84912024-01-24T19:09:05.851Z 11 220-239 78 76 78 127 125 127 F F F <---
84922024-01-24T19:09:05.851Z 12 240-259 78 76 78 127 125 127 T F T <---
84932024-01-24T19:09:05.851Z 13 260-279 78 77 78 127 126 127 T F T <---
84942024-01-24T19:09:05.852Z 14 280-299 78 77 78 129 126 129 F F F <---
84952024-01-24T19:09:05.852Z 15 300-319 78 74 78 127 120 127 F F F <---
84962024-01-24T19:09:05.852Z 16 320-339 74 74 74 121 121 121 F F F
84972024-01-24T19:09:05.852Z 17 340-359 76 76 76 125 125 125 F F F
84982024-01-24T19:09:05.852Z 18 360-379 77 77 77 126 126 126 F F F
84992024-01-24T19:09:05.852Z 19 380-399 77 77 77 126 126 126 F F F
85002024-01-24T19:09:05.872Z 20 400-419 76 76 76 125 125 125 F F F
85012024-01-24T19:09:05.872Z 21 420-439 77 77 77 126 126 126 F F F
85022024-01-24T19:09:05.872Z 22 440-459 78 77 78 127 126 127 F T F <---
85032024-01-24T19:09:05.872Z 23 460-479 75 75 75 122 122 122 F F F
85042024-01-24T19:09:05.872Z 24 480-499 77 77 77 126 126 126 F F F
85052024-01-24T19:09:05.872Z 25 500-519 76 76 76 124 124 124 F F F
85062024-01-24T19:09:05.872Z 26 520-539 72 72 72 118 118 118 T F T <---
85072024-01-24T19:09:05.872Z 27 540-559 74 74 74 120 120 120 F F F
85082024-01-24T19:09:05.873Z 28 560-579 77 77 77 126 126 126 F F F
85092024-01-24T19:09:05.873Z 29 580-599 72 72 72 118 118 118 F F F
85102024-01-24T19:09:05.873ZMax gen: 78, Max flush: 129
85112024-01-24T19:09:05.873ZError: Difference in extent metadata found!
85122024-01-24T19:09:05.873ZA Difference in extent metadata is expected here
85132024-01-24T19:09:05.873ZOn loop 39
85142024-01-24T19:09:05.873Z
85152024-01-24T19:09:05.873ZVerifying data now
85162024-01-24T19:09:10.933ZNo change after verify
85172024-01-24T19:09:10.934ZLoop: 39 Downstairs dump after verify (and repair):
85182024-01-24T19:09:10.959ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85192024-01-24T19:09:10.959Z 0 000-019 70 70 70 116 116 116 F F F
85202024-01-24T19:09:10.959Z 1 020-039 76 76 76 124 124 124 F F F
85212024-01-24T19:09:10.959Z 2 040-059 78 78 78 128 128 128 F F F
85222024-01-24T19:09:10.959Z 3 060-079 77 77 77 126 126 126 F F F
85232024-01-24T19:09:10.960Z 4 080-099 72 72 72 118 118 118 F F F
85242024-01-24T19:09:10.960Z 5 100-119 78 78 78 127 127 127 F F F
85252024-01-24T19:09:10.961Z 6 120-139 79 79 79 130 130 130 F F F
85262024-01-24T19:09:10.961Z 7 140-159 75 75 75 122 122 122 F F F
85272024-01-24T19:09:10.962Z 8 160-179 78 78 78 127 127 127 F F F
85282024-01-24T19:09:10.962Z 9 180-199 72 72 72 118 118 118 F F F
85292024-01-24T19:09:10.962Z 10 200-219 79 79 79 130 130 130 F F F
85302024-01-24T19:09:10.962Z 11 220-239 78 78 78 127 127 127 F F F
85312024-01-24T19:09:10.962Z 12 240-259 79 79 79 130 130 130 F F F
85322024-01-24T19:09:10.962Z 13 260-279 79 79 79 130 130 130 F F F
85332024-01-24T19:09:10.962Z 14 280-299 78 78 78 129 129 129 F F F
85342024-01-24T19:09:10.962Z 15 300-319 78 78 78 127 127 127 F F F
85352024-01-24T19:09:10.963Z 16 320-339 74 74 74 121 121 121 F F F
85362024-01-24T19:09:10.963Z 17 340-359 76 76 76 125 125 125 F F F
85372024-01-24T19:09:10.964Z 18 360-379 77 77 77 126 126 126 F F F
85382024-01-24T19:09:10.964Z 19 380-399 77 77 77 126 126 126 F F F
85392024-01-24T19:09:10.964Z 20 400-419 76 76 76 125 125 125 F F F
85402024-01-24T19:09:10.965Z 21 420-439 77 77 77 126 126 126 F F F
85412024-01-24T19:09:10.965Z 22 440-459 78 78 78 127 127 127 F F F
85422024-01-24T19:09:10.966Z 23 460-479 75 75 75 122 122 122 F F F
85432024-01-24T19:09:10.966Z 24 480-499 77 77 77 126 126 126 F F F
85442024-01-24T19:09:10.966Z 25 500-519 76 76 76 124 124 124 F F F
85452024-01-24T19:09:10.966Z 26 520-539 79 79 79 130 130 130 F F F
85462024-01-24T19:09:10.966Z 27 540-559 74 74 74 120 120 120 F F F
85472024-01-24T19:09:10.966Z 28 560-579 77 77 77 126 126 126 F F F
85482024-01-24T19:09:10.967Z 29 580-599 72 72 72 118 118 118 F F F
85492024-01-24T19:09:10.967ZMax gen: 79, Max flush: 130
85502024-01-24T19:09:10.967Z
85512024-01-24T19:09:10.968ZBegin loop 40
85522024-01-24T19:09:10.969ZDownstairs to restart: 0
85532024-01-24T19:09:10.969Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.551586573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449}
85542024-01-24T19:09:10.969Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552064641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449}
85552024-01-24T19:09:10.970Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552110599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449}
85562024-01-24T19:09:10.970Z{"msg":"Crucible 4142f27e-fbc9-4b48-81a7-e7e5f67e4ebc has session id: 4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552178889Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85572024-01-24T19:09:10.970Z{"msg":"Upstairs opts: Upstairs UUID: 4142f27e-fbc9-4b48-81a7-e7e5f67e4ebc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552234346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85582024-01-24T19:09:10.970Z{"msg":"Crucible stats registered with UUID: 4142f27e-fbc9-4b48-81a7-e7e5f67e4ebc","v":0,"name":"crucible","level":30Crucible runtime is spawned
85592024-01-24T19:09:10.970Z,"time":"2024-01-24T19:09:09.552306238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85602024-01-24T19:09:10.970Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552546423Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449}
85612024-01-24T19:09:10.971Z{"msg":"4142f27e-fbc9-4b48-81a7-e7e5f67e4ebc active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552868674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85622024-01-24T19:09:10.971Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552914808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85632024-01-24T19:09:10.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552939205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85642024-01-24T19:09:10.972Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.552961704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85652024-01-24T19:09:10.972Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553045379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85662024-01-24T19:09:10.973Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553092349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85672024-01-24T19:09:10.973Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553140884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85682024-01-24T19:09:10.973Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55318531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85692024-01-24T19:09:10.973Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55343677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85702024-01-24T19:09:10.973Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553494255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85712024-01-24T19:09:10.973Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553540782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85722024-01-24T19:09:10.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553583977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85732024-01-24T19:09:10.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.553626585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85742024-01-24T19:09:10.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55475837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85752024-01-24T19:09:10.976Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55491195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85762024-01-24T19:09:10.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.555002796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85772024-01-24T19:09:10.976Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.555112643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85782024-01-24T19:09:10.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.555181163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85792024-01-24T19:09:10.977Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.555213385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85802024-01-24T19:09:10.977Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.555903564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85812024-01-24T19:09:10.977Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.556140252Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85822024-01-24T19:09:10.977Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.556374173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85832024-01-24T19:09:10.979Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557044487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85842024-01-24T19:09:10.979Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557114912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85852024-01-24T19:09:10.979Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557161733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85862024-01-24T19:09:10.979Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557216603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85872024-01-24T19:09:10.979Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557259379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85882024-01-24T19:09:10.979Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557353819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85892024-01-24T19:09:10.980Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55739483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85902024-01-24T19:09:10.980Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557448266Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85912024-01-24T19:09:10.980Z{"msg":"[0]R flush_numbers[0..12]: [116, 124, 128, 126, 118, 127, 130, 122, 127, 118, 130, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557476283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85922024-01-24T19:09:10.980Z{"msg":"[0]R generation[0..12]: [70, 76, 78, 77, 72, 78, 79, 75, 78, 72, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557520904Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85932024-01-24T19:09:10.981Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557548447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85942024-01-24T19:09:10.982Z{"msg":"[1]R flush_numbers[0..12]: [116, 124, 128, 126, 118, 127, 130, 122, 127, 118, 130, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557592696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85952024-01-24T19:09:10.982Z{"msg":"[1]R generation[0..12]: [70, 76, 78, 77, 72, 78, 79, 75, 78, 72, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557619593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85962024-01-24T19:09:10.982ZWait for a query_work_queue command to finish before sending IO
85972024-01-24T19:09:10.982Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557669653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85982024-01-24T19:09:10.982Z{"msg":"[2]R flush_numbers[0..12]: [116, 124, 128, 126, 118, 127, 130, 122, 127, 118, 130, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557697749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
85992024-01-24T19:09:10.983Z{"msg":"[2]R generation[0..12]: [70, 76, 78, 77, 72, 78, 79, 75, 78, 72, 79, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557747937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86002024-01-24T19:09:10.983Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557775399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86012024-01-24T19:09:10.983Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557818324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86022024-01-24T19:09:10.983Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557844594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86032024-01-24T19:09:10.984Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557899946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86042024-01-24T19:09:10.985Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557935226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86052024-01-24T19:09:10.985Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.557963278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86062024-01-24T19:09:10.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558005691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86072024-01-24T19:09:10.985Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558031834Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86082024-01-24T19:09:10.986Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558075033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86092024-01-24T19:09:10.986Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.55810095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86102024-01-24T19:09:10.986Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558143592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86112024-01-24T19:09:10.986Z{"msg":"4142f27e-fbc9-4b48-81a7-e7e5f67e4ebc is now active with session: 4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558169273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86122024-01-24T19:09:10.986Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86132024-01-24T19:09:10.986Z,"time":"2024-01-24T19:09:09.558214045Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449,"session_id":"4c2f3f3b-0c04-46c0-a58e-038d4c57b9ca"}
86142024-01-24T19:09:10.987Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:09.558239124Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1449}
86152024-01-24T19:09:10.988ZLoading write count information from file "/tmp/test_repair_verify.data"
86162024-01-24T19:09:10.988ZRun Repair workload
86172024-01-24T19:09:10.988Z01/30 Flush
86182024-01-24T19:09:11.019Z02/30 Write block 539 len 36864 data: 7 5 7 9 9 9 8 7 9
86192024-01-24T19:09:11.020Z03/30 Write block 25 len 28672 data: 7 8 9 10 9 9 10
86202024-01-24T19:09:11.020Z04/30 Read block 342 len 16384
86212024-01-24T19:09:11.020Z05/30 Read block 28 len 12288
86222024-01-24T19:09:11.020Z06/30 Read block 225 len 36864
86232024-01-24T19:09:11.020Z07/30 Read block 366 len 24576
86242024-01-24T19:09:11.020Z08/30 Write block 130 len 40960 data: 8 8 8 5 7 7 9 9 7 8
86252024-01-24T19:09:11.020Z09/30 Read block 9 len 28672
86262024-01-24T19:09:11.020Z10/30 Write block 394 len 12288 data: 9 7 8
86272024-01-24T19:09:11.021Z11/30 Read block 204 len 20480
86282024-01-24T19:09:11.021Z12/30 Write block 533 len 8192 data: 6 5
86292024-01-24T19:09:11.021Z13/30 Read block 509 len 28672
86302024-01-24T19:09:11.021Z14/30 Write block 501 len 8192 data: 6 4
86312024-01-24T19:09:11.021Z15/30 Read block 131 len 36864
86322024-01-24T19:09:11.021Z16/30 Write block 155 len 16384 data: 6 6 7 6
86332024-01-24T19:09:11.021Z17/30 Flush
86342024-01-24T19:09:11.021Z18/30 Write block 361 len 24576 data: 9 8 7 9 9 9
86352024-01-24T19:09:11.022Z19/30 Write block 568 len 36864 data: 6 6 4 6 7 6 6 7 7
86362024-01-24T19:09:11.022Z20/30 Write block 172 len 40960 data: 9 7 9 8 8 6 6 5 7 7
86372024-01-24T19:09:11.022Z21/30 Read block 316 len 24576
86382024-01-24T19:09:11.022Z22/30 Flush
86392024-01-24T19:09:11.022Z23/30 Write block 16 len 16384 data: 3 4 3 5
86402024-01-24T19:09:11.022Z24/30 Read block 591 len 4096
86412024-01-24T19:09:11.022Z25/30 Write block 183 len 12288 data: 6 8 9
86422024-01-24T19:09:11.023Z26/30 Write block 50 len 32768 data: 10 7 6 6 6 7 9 9
86432024-01-24T19:09:11.023Z27/30 Write block 215 len 8192 data: 5 7
86442024-01-24T19:09:11.023Z28/30 Write block 567 len 4096 data: 6
86452024-01-24T19:09:11.023Z29/30 Write block 145 len 24576 data: 4 6 7 5 7 7
86462024-01-24T19:09:11.023Z30/30 Write block 158 len 16384 data: 7 5 5 10
86472024-01-24T19:09:11.023Z----------------------------------------------------------------
86482024-01-24T19:09:11.024Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
86492024-01-24T19:09:11.024ZStates: Active Active Active
86502024-01-24T19:09:11.024ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86512024-01-24T19:09:11.024Z 1 Acked 1000 Flush 0 Sent Done Done false
86522024-01-24T19:09:11.024Z 2 Acked 1001 Write 9 Sent Done Done false
86532024-01-24T19:09:11.024Z 3 Acked 1002 Write 7 Sent Done Done false
86542024-01-24T19:09:11.024Z 4 Acked 1003 Read 4 Sent Done Done false
86552024-01-24T19:09:11.025Z 5 Acked 1004 Read 3 Sent Done Done false
86562024-01-24T19:09:11.025Z 6 Acked 1005 Read 9 Sent Done Done false
86572024-01-24T19:09:11.025Z 7 Acked 1006 Read 6 Sent Done Done false
86582024-01-24T19:09:11.025Z 8 Acked 1007 Write 10 Sent Done Done false
86592024-01-24T19:09:11.025Z 9 Acked 1008 Read 7 Sent Done Done false
86602024-01-24T19:09:11.025Z 10 Acked 1009 Write 3 Sent Done Done false
86612024-01-24T19:09:11.025Z 11 Acked 1010 Read 5 Sent Done Done false
86622024-01-24T19:09:11.025Z 12 Acked 1011 Write 2 Sent Done Done false
86632024-01-24T19:09:11.025Z 13 Acked 1012 Read 7 Sent Done Done false
86642024-01-24T19:09:11.025Z 14 Acked 1013 Write 2 Sent Done Done false
86652024-01-24T19:09:11.026Z 15 Acked 1014 Read 9 Sent Done Done false
86662024-01-24T19:09:11.026Z 16 Acked 1015 Write 4 Sent Done Done false
86672024-01-24T19:09:11.026Z 17 Acked 1016 Flush 0 Sent Done Done false
86682024-01-24T19:09:11.026Z 18 Acked 1017 Write 6 Sent Done Done false
86692024-01-24T19:09:11.026Z 19 Acked 1018 Write 9 Sent Done Done false
86702024-01-24T19:09:11.027Z 20 Acked 1019 Write 10 Sent Done Done false
86712024-01-24T19:09:11.027Z 21 Acked 1020 Read 6 Sent Done Done false
86722024-01-24T19:09:11.027Z 22 Acked 1021 Flush 0 Sent Done Done false
86732024-01-24T19:09:11.027Z 23 Acked 1022 Write 4 Sent Done Done false
86742024-01-24T19:09:11.027Z 24 Acked 1023 Read 1 Sent Done Done false
86752024-01-24T19:09:11.027Z 25 Acked 1024 Write 3 Sent Done Done false
86762024-01-24T19:09:11.027Z 26 Acked 1025 Write 8 Sent Done Done false
86772024-01-24T19:09:11.027Z 27 Acked 1026 Write 2 Sent Done Done false
86782024-01-24T19:09:11.027Z 28 Acked 1027 Write 1 Sent Done Done false
86792024-01-24T19:09:11.028Z 29 Acked 1028 Write 6 Sent Sent Sent false
86802024-01-24T19:09:11.028Z 30 Acked 1029 Write 4 Sent Sent Sent false
86812024-01-24T19:09:11.028Z STATES DS:0 DS:1 DS:2 TOTAL
86822024-01-24T19:09:11.028Z New 0 0 0 0
86832024-01-24T19:09:11.028Z Sent 30 2 2 34
86842024-01-24T19:09:11.028Z Done 0 28 28 56
86852024-01-24T19:09:11.029Z Skipped 0 0 0 0
86862024-01-24T19:09:11.029Z Error 0 0 0 0
86872024-01-24T19:09:11.029ZLast Flush: 0 1021 1021
86882024-01-24T19:09:11.029ZDownstairs last five completed:
86892024-01-24T19:09:11.029ZUpstairs last five completed: 30 29 28 27 26
86902024-01-24T19:09:11.029ZWrote out file "/tmp/test_repair_verify.data"
86912024-01-24T19:09:11.029Z
86922024-01-24T19:09:11.029ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
86932024-01-24T19:09:11.030ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86942024-01-24T19:09:11.030Z 0 000-019 70 70 70 116 116 116 F T T <---
86952024-01-24T19:09:11.030Z 1 020-039 76 80 80 124 132 132 F F F <---
86962024-01-24T19:09:11.030Z 2 040-059 78 78 78 128 128 128 F T T <---
86972024-01-24T19:09:11.030Z 3 060-079 77 77 77 126 126 126 F F F
86982024-01-24T19:09:11.030Z 4 080-099 72 72 72 118 118 118 F F F
86992024-01-24T19:09:11.030Z 5 100-119 78 78 78 127 127 127 F F F
87002024-01-24T19:09:11.030Z 6 120-139 79 80 80 130 132 132 F F F <---
87012024-01-24T19:09:11.030Z 7 140-159 75 80 80 122 132 132 F T T <---
87022024-01-24T19:09:11.030Z 8 160-179 78 80 80 127 133 133 F T T <---
87032024-01-24T19:09:11.030Z 9 180-199 72 80 80 118 133 133 F T T <---
87042024-01-24T19:09:11.030Z 10 200-219 79 79 79 130 130 130 F T T <---
87052024-01-24T19:09:11.031Z 11 220-239 78 78 78 127 127 127 F F F
87062024-01-24T19:09:11.031Z 12 240-259 79 79 79 130 130 130 F F F
87072024-01-24T19:09:11.031Z 13 260-279 79 79 79 130 130 130 F F F
87082024-01-24T19:09:11.031Z 14 280-299 78 78 78 129 129 129 F F F
87092024-01-24T19:09:11.033Z 15 300-319 78 78 78 127 127 127 F F F
87102024-01-24T19:09:11.033Z 16 320-339 74 74 74 121 121 121 F F F
87112024-01-24T19:09:11.033Z 17 340-359 76 76 76 125 125 125 F F F
87122024-01-24T19:09:11.033Z 18 360-379 77 80 80 126 133 133 F F F <---
87132024-01-24T19:09:11.033Z 19 380-399 77 80 80 126 132 132 F F F <---
87142024-01-24T19:09:11.033Z 20 400-419 76 76 76 125 125 125 F F F
87152024-01-24T19:09:11.033Z 21 420-439 77 77 77 126 126 126 F F F
87162024-01-24T19:09:11.033Z 22 440-459 78 78 78 127 127 127 F F F
87172024-01-24T19:09:11.033Z 23 460-479 75 75 75 122 122 122 F F F
87182024-01-24T19:09:11.043Z 24 480-499 77 77 77 126 126 126 F F F
87192024-01-24T19:09:11.043Z 25 500-519 76 80 80 124 132 132 F F F <---
87202024-01-24T19:09:11.043Z 26 520-539 79 80 80 130 132 132 F F F <---
87212024-01-24T19:09:11.043Z 27 540-559 74 80 80 120 132 132 F F F <---
87222024-01-24T19:09:11.044Z 28 560-579 77 80 80 126 133 133 F T T <---
87232024-01-24T19:09:11.044Z 29 580-599 72 72 72 118 118 118 F F F
87242024-01-24T19:09:11.044ZMax gen: 80, Max flush: 133
87252024-01-24T19:09:11.044ZError: Difference in extent metadata found!
87262024-01-24T19:09:11.044ZA Difference in extent metadata is expected here
87272024-01-24T19:09:11.044ZOn loop 40
87282024-01-24T19:09:11.044Z
87292024-01-24T19:09:11.044ZVerifying data now
87302024-01-24T19:09:16.635ZNo change after verify
87312024-01-24T19:09:16.636ZLoop: 40 Downstairs dump after verify (and repair):
87322024-01-24T19:09:16.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87332024-01-24T19:09:16.644Z 0 000-019 81 81 81 134 134 134 F F F
87342024-01-24T19:09:16.644Z 1 020-039 80 80 80 132 132 132 F F F
87352024-01-24T19:09:16.644Z 2 040-059 81 81 81 134 134 134 F F F
87362024-01-24T19:09:16.644Z 3 060-079 77 77 77 126 126 126 F F F
87372024-01-24T19:09:16.644Z 4 080-099 72 72 72 118 118 118 F F F
87382024-01-24T19:09:16.644Z 5 100-119 78 78 78 127 127 127 F F F
87392024-01-24T19:09:16.644Z 6 120-139 80 80 80 132 132 132 F F F
87402024-01-24T19:09:16.644Z 7 140-159 81 81 81 134 134 134 F F F
87412024-01-24T19:09:16.644Z 8 160-179 81 81 81 134 134 134 F F F
87422024-01-24T19:09:16.645Z 9 180-199 81 81 81 134 134 134 F F F
87432024-01-24T19:09:16.645Z 10 200-219 81 81 81 134 134 134 F F F
87442024-01-24T19:09:16.645Z 11 220-239 78 78 78 127 127 127 F F F
87452024-01-24T19:09:16.645Z 12 240-259 79 79 79 130 130 130 F F F
87462024-01-24T19:09:16.645Z 13 260-279 79 79 79 130 130 130 F F F
87472024-01-24T19:09:16.645Z 14 280-299 78 78 78 129 129 129 F F F
87482024-01-24T19:09:16.645Z 15 300-319 78 78 78 127 127 127 F F F
87492024-01-24T19:09:16.645Z 16 320-339 74 74 74 121 121 121 F F F
87502024-01-24T19:09:16.645Z 17 340-359 76 76 76 125 125 125 F F F
87512024-01-24T19:09:16.645Z 18 360-379 80 80 80 133 133 133 F F F
87522024-01-24T19:09:16.646Z 19 380-399 80 80 80 132 132 132 F F F
87532024-01-24T19:09:16.646Z 20 400-419 76 76 76 125 125 125 F F F
87542024-01-24T19:09:16.646Z 21 420-439 77 77 77 126 126 126 F F F
87552024-01-24T19:09:16.646Z 22 440-459 78 78 78 127 127 127 F F F
87562024-01-24T19:09:16.646Z 23 460-479 75 75 75 122 122 122 F F F
87572024-01-24T19:09:16.646Z 24 480-499 77 77 77 126 126 126 F F F
87582024-01-24T19:09:16.647Z 25 500-519 80 80 80 132 132 132 F F F
87592024-01-24T19:09:16.647Z 26 520-539 80 80 80 132 132 132 F F F
87602024-01-24T19:09:16.647Z 27 540-559 80 80 80 132 132 132 F F F
87612024-01-24T19:09:16.647Z 28 560-579 81 81 81 134 134 134 F F F
87622024-01-24T19:09:16.647Z 29 580-599 72 72 72 118 118 118 F F F
87632024-01-24T19:09:16.648ZMax gen: 81, Max flush: 134
87642024-01-24T19:09:16.648Z
87652024-01-24T19:09:16.648ZBegin loop 41
87662024-01-24T19:09:16.648ZDownstairs to restart: 2
87672024-01-24T19:09:16.648Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938126323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457}
87682024-01-24T19:09:16.648Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938597785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457}
87692024-01-24T19:09:16.649Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938635256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457}
87702024-01-24T19:09:16.649Z{"msg":"Crucible ef85b3ba-c3f8-4088-8212-61837927e901 has session id: 27bca4df-6f2c-444e-a009-0956b05ea478","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938673465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87712024-01-24T19:09:16.649Z{"msg":"Upstairs opts: Upstairs UUID: ef85b3ba-c3f8-4088-8212-61837927e901, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938759716Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87722024-01-24T19:09:16.649Z{"msg":"Crucible stats registered with UUID: ef85b3ba-c3f8-4088-8212-61837927e901","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.938801262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87732024-01-24T19:09:16.650ZCrucible runtime is spawned
87742024-01-24T19:09:16.650Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939128516Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457}
87752024-01-24T19:09:16.650Z{"msg":"ef85b3ba-c3f8-4088-8212-61837927e901 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939391965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87762024-01-24T19:09:16.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939522162Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87772024-01-24T19:09:16.651Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939596958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87782024-01-24T19:09:16.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939671733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87792024-01-24T19:09:16.651Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939719361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87802024-01-24T19:09:16.651Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939791792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87812024-01-24T19:09:16.652Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.939895918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87822024-01-24T19:09:16.652Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940039668Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87832024-01-24T19:09:16.652Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940146219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87842024-01-24T19:09:16.652Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940226534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87852024-01-24T19:09:16.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940274765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87862024-01-24T19:09:16.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940320893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87872024-01-24T19:09:16.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.940460758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87882024-01-24T19:09:16.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.941258303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87892024-01-24T19:09:16.653Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.941352516Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87902024-01-24T19:09:16.653Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94140417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87912024-01-24T19:09:16.654Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94144829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87922024-01-24T19:09:16.654Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.941526177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87932024-01-24T19:09:16.654Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.941575073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87942024-01-24T19:09:16.654Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.942341716Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87952024-01-24T19:09:16.654Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.942437454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87962024-01-24T19:09:16.654Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.942487311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87972024-01-24T19:09:16.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943220517Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87982024-01-24T19:09:16.655Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943314282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
87992024-01-24T19:09:16.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943362813Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88002024-01-24T19:09:16.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943442464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88012024-01-24T19:09:16.656Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94349165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88022024-01-24T19:09:16.657Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94356725Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88032024-01-24T19:09:16.657Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943637322Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88042024-01-24T19:09:16.657Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943711158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88052024-01-24T19:09:16.657Z{"msg":"[0]R flush_numbers[0..12]: [134, 132, 134, 126, 118, 127, 132, 134, 134, 134, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943786066Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88062024-01-24T19:09:16.657Z{"msg":"[0]R generation[0..12]: [81, 80, 81, 77, 72, 78, 80, 81, 81, 81, 81, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.943858928Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88072024-01-24T19:09:16.657Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
88082024-01-24T19:09:16.657Zcrucible","level":30,"time":"2024-01-24T19:09:14.943935265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88092024-01-24T19:09:16.657Z{"msg":"[1]R flush_numbers[0..12]: [134, 132, 134, 126, 118, 127, 132, 134, 134, 134, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944017353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88102024-01-24T19:09:16.657Z{"msg":"[1]R generation[0..12]: [81, 80, 81, 77, 72, 78, 80, 81, 81, 81, 81, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944066937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88112024-01-24T19:09:16.658Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944140159Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88122024-01-24T19:09:16.658Z{"msg":"[2]R flush_numbers[0..12]: [134, 132, 134, 126, 118, 127, 132, 134, 134, 134, 134, 127]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944221161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88132024-01-24T19:09:16.658Z{"msg":"[2]R generation[0..12]: [81, 80, 81, 77, 72, 78, 80, 81, 81, 81, 81, 78]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944300894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88142024-01-24T19:09:16.658Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944378078Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88152024-01-24T19:09:16.658Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944450602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88162024-01-24T19:09:16.658Z{"msg":"Generation requested: 82 >= found:82","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88172024-01-24T19:09:16.659Z,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944527093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88182024-01-24T19:09:16.659Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94460444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88192024-01-24T19:09:16.659Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944711327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88202024-01-24T19:09:16.659Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944746094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88212024-01-24T19:09:16.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944789737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88222024-01-24T19:09:16.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944815144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88232024-01-24T19:09:16.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944858581Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88242024-01-24T19:09:16.659Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944883413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88252024-01-24T19:09:16.659Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944924461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88262024-01-24T19:09:16.659Z{"msg":"ef85b3ba-c3f8-4088-8212-61837927e901 is now active with session: 27bca4df-6f2c-444e-a009-0956b05ea478","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.94494925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88272024-01-24T19:09:16.659Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.944970439Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457,"session_id":"27bca4df-6f2c-444e-a009-0956b05ea478"}
88282024-01-24T19:09:16.659Z{"msg":"Loading write count information from file "/tmp/test_repair_verify.data"
88292024-01-24T19:09:16.659ZThe guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:14.945027106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1457}
88302024-01-24T19:09:16.659ZRun Repair workload
88312024-01-24T19:09:16.659Z01/30 Write block 105 len 20480 data: 8 8 9 10 8
88322024-01-24T19:09:16.685Z02/30 Read block 38 len 32768
88332024-01-24T19:09:16.685Z03/30 Read block 224 len 40960
88342024-01-24T19:09:16.685Z04/30 Write block 163 len 4096 data: 11
88352024-01-24T19:09:16.685Z05/30 Read block 541 len 4096
88362024-01-24T19:09:16.685Z06/30 Write block 192 len 24576 data: 6 6 6 5 5 5
88372024-01-24T19:09:16.685Z07/30 Read block 362 len 16384
88382024-01-24T19:09:16.685Z08/30 Read block 7 len 32768
88392024-01-24T19:09:16.685Z09/30 Read block 309 len 12288
88402024-01-24T19:09:16.685Z10/30 Write block 316 len 40960 data: 9 9 9 9 9 9 8 9 8 10
88412024-01-24T19:09:16.685Z11/30 Write block 501 len 20480 data: 7 5 2 4 4
88422024-01-24T19:09:16.686Z12/30 Flush
88432024-01-24T19:09:16.686Z13/30 Write block 395 len 16384 data: 8 9 6 6
88442024-01-24T19:09:16.686Z14/30 Read block 567 len 4096
88452024-01-24T19:09:16.686Z15/30 Flush
88462024-01-24T19:09:16.686Z16/30 Write block 135 len 24576 data: 8 10 10 8 9 7
88472024-01-24T19:09:16.686Z17/30 Read block 263 len 28672
88482024-01-24T19:09:16.686Z18/30 Read block 589 len 4096
88492024-01-24T19:09:16.686Z19/30 Read block 393 len 36864
88502024-01-24T19:09:16.686Z20/30 Write block 550 len 40960 data: 6 6 5 4 3 2 4 6 6 6
88512024-01-24T19:09:16.686Z21/30 Read block 548 len 28672
88522024-01-24T19:09:16.686Z22/30 Read block 174 len 36864
88532024-01-24T19:09:16.686Z23/30 Write block 216 len 24576 data: 8 7 7 6 5 6
88542024-01-24T19:09:16.686Z24/30 Write block 510 len 16384 data: 10 11 11 10
88552024-01-24T19:09:16.686Z25/30 Read block 9 len 20480
88562024-01-24T19:09:16.687Z26/30 Read block 204 len 24576
88572024-01-24T19:09:16.687Z27/30 Write block 59 len 36864 data: 7 6 5 4 5 7 6 3 3
88582024-01-24T19:09:16.687Z28/30 Write block 381 len 20480 data: 6 4 5 4 3
88592024-01-24T19:09:16.687Z29/30 Write block 493 len 4096 data: 7
88602024-01-24T19:09:16.687Z30/30 Read block 11 len 4096
88612024-01-24T19:09:16.688Z----------------------------------------------------------------
88622024-01-24T19:09:16.688Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
88632024-01-24T19:09:16.688ZStates: Active Active Active
88642024-01-24T19:09:16.688ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88652024-01-24T19:09:16.688Z 1 Acked 1000 Write 5 Done Done Done false
88662024-01-24T19:09:16.688Z 2 Acked 1001 Read 8 Done Done Done false
88672024-01-24T19:09:16.688Z 3 Acked 1002 Read 10 Done Done Done false
88682024-01-24T19:09:16.688Z 4 Acked 1003 Write 1 Done Done Done false
88692024-01-24T19:09:16.688Z 5 Acked 1004 Read 1 Done Done Done false
88702024-01-24T19:09:16.688Z 6 Acked 1005 Write 6 Done Done Done false
88712024-01-24T19:09:16.688Z 7 Acked 1006 Read 4 Done Done Done false
88722024-01-24T19:09:16.688Z 8 Acked 1007 Read 8 Done Done Sent false
88732024-01-24T19:09:16.688Z 9 Acked 1008 Read 3 Done Done Sent false
88742024-01-24T19:09:16.688Z 10 Acked 1009 Write 10 Done Done Sent false
88752024-01-24T19:09:16.689Z 11 Acked 1010 Write 5 Done Done Sent false
88762024-01-24T19:09:16.689Z 12 Acked 1011 Flush 0 Done Done Sent false
88772024-01-24T19:09:16.689Z 13 Acked 1012 Write 4 Done Done Sent false
88782024-01-24T19:09:16.689Z 14 Acked 1013 Read 1 Done Done Sent false
88792024-01-24T19:09:16.689Z 15 Acked 1014 Flush 0 Done Done Sent false
88802024-01-24T19:09:16.689Z 16 Acked 1015 Write 6 Done Done Sent false
88812024-01-24T19:09:16.689Z 17 Acked 1016 Read 7 Done Done Sent false
88822024-01-24T19:09:16.689Z 18 Acked 1017 Read 1 Done Done Sent false
88832024-01-24T19:09:16.689Z 19 Acked 1018 Read 9 Done Done Sent false
88842024-01-24T19:09:16.689Z 20 Acked 1019 Write 10 Done Done Sent false
88852024-01-24T19:09:16.689Z 21 Acked 1020 Read 7 Done Done Sent false
88862024-01-24T19:09:16.689Z 22 Acked 1021 Read 9 Done Done Sent false
88872024-01-24T19:09:16.689Z 23 Acked 1022 Write 6 Done Done Sent false
88882024-01-24T19:09:16.689Z 24 Acked 1023 Write 4 Done Done Sent false
88892024-01-24T19:09:16.689Z 25 Acked 1024 Read 5 Done Done Sent false
88902024-01-24T19:09:16.690Z 26 Acked 1025 Read 6 Done Done Sent false
88912024-01-24T19:09:16.690Z 27 Acked 1026 Write 9 Done Done Sent false
88922024-01-24T19:09:16.690Z 28 Acked 1027 Write 5 Done Done Sent false
88932024-01-24T19:09:16.690Z 29 Acked 1028 Write 1 Done Done Sent false
88942024-01-24T19:09:16.690Z 30 Acked 1029 Read 1 Done Done Sent false
88952024-01-24T19:09:16.690Z STATES DS:0 DS:1 DS:2 TOTAL
88962024-01-24T19:09:16.690Z New 0 0 0 0
88972024-01-24T19:09:16.690Z Sent 0 0 23 23
88982024-01-24T19:09:16.690Z Done 30 30 7 67
88992024-01-24T19:09:16.690Z Skipped 0 0 0 0
89002024-01-24T19:09:16.690Z Error 0 0 0 0
89012024-01-24T19:09:16.690ZLast Flush: 1014 1014 0
89022024-01-24T19:09:16.690ZDownstairs last five completed:
89032024-01-24T19:09:16.690ZUpstairs last five completed: 30 29 28 27 26
89042024-01-24T19:09:16.690ZWrote out file "/tmp/test_repair_verify.data"
89052024-01-24T19:09:16.690Z
89062024-01-24T19:09:16.690ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89072024-01-24T19:09:16.690ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89082024-01-24T19:09:16.691Z 0 000-019 81 81 81 134 134 134 F F F
89092024-01-24T19:09:16.691Z 1 020-039 80 80 80 132 132 132 F F F
89102024-01-24T19:09:16.691Z 2 040-059 81 81 81 134 134 134 T T F <---
89112024-01-24T19:09:16.691Z 3 060-079 77 77 77 126 126 126 T T F <---
89122024-01-24T19:09:16.691Z 4 080-099 72 72 72 118 118 118 F F F
89132024-01-24T19:09:16.691Z 5 100-119 82 82 78 135 135 127 F F T <---
89142024-01-24T19:09:16.691Z 6 120-139 80 80 80 132 132 132 T T F <---
89152024-01-24T19:09:16.691Z 7 140-159 81 81 81 134 134 134 T T F <---
89162024-01-24T19:09:16.691Z 8 160-179 82 82 81 135 135 134 F F T <---
89172024-01-24T19:09:16.691Z 9 180-199 82 82 81 135 135 134 F F T <---
89182024-01-24T19:09:16.691Z 10 200-219 81 81 81 134 134 134 T T F <---
89192024-01-24T19:09:16.691Z 11 220-239 78 78 78 127 127 127 T T F <---
89202024-01-24T19:09:16.691Z 12 240-259 79 79 79 130 130 130 F F F
89212024-01-24T19:09:16.691Z 13 260-279 79 79 79 130 130 130 F F F
89222024-01-24T19:09:16.692Z 14 280-299 78 78 78 129 129 129 F F F
89232024-01-24T19:09:16.692Z 15 300-319 82 82 78 135 135 127 F F F <---
89242024-01-24T19:09:16.692Z 16 320-339 82 82 74 135 135 121 F F F <---
89252024-01-24T19:09:16.692Z 17 340-359 76 76 76 125 125 125 F F F
89262024-01-24T19:09:16.692Z 18 360-379 80 80 80 133 133 133 F F F
89272024-01-24T19:09:16.692Z 19 380-399 82 82 80 136 136 132 T T F <---
89282024-01-24T19:09:16.692Z 20 400-419 76 76 76 125 125 125 F F F
89292024-01-24T19:09:16.692Z 21 420-439 77 77 77 126 126 126 F F F
89302024-01-24T19:09:16.692Z 22 440-459 78 78 78 127 127 127 F F F
89312024-01-24T19:09:16.692Z 23 460-479 75 75 75 122 122 122 F F F
89322024-01-24T19:09:16.696Z 24 480-499 77 77 77 126 126 126 T T F <---
89332024-01-24T19:09:16.696Z 25 500-519 82 82 80 135 135 132 T T F <---
89342024-01-24T19:09:16.696Z 26 520-539 80 80 80 132 132 132 F F F
89352024-01-24T19:09:16.696Z 27 540-559 80 80 80 132 132 132 T T F <---
89362024-01-24T19:09:16.696Z 28 560-579 81 81 81 134 134 134 F F F
89372024-01-24T19:09:16.696Z 29 580-599 72 72 72 118 118 118 F F F
89382024-01-24T19:09:16.696ZMax gen: 82, Max flush: 136
89392024-01-24T19:09:16.696ZError: Difference in extent metadata found!
89402024-01-24T19:09:16.696ZA Difference in extent metadata is expected here
89412024-01-24T19:09:16.696ZOn loop 41
89422024-01-24T19:09:16.696Z
89432024-01-24T19:09:16.696ZVerifying data now
89442024-01-24T19:09:22.700ZNo change after verify
89452024-01-24T19:09:22.701ZLoop: 41 Downstairs dump after verify (and repair):
89462024-01-24T19:09:22.737ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89472024-01-24T19:09:22.737Z 0 000-019 81 81 81 134 134 134 F F F
89482024-01-24T19:09:22.737Z 1 020-039 80 80 80 132 132 132 F F F
89492024-01-24T19:09:22.737Z 2 040-059 83 83 83 137 137 137 F F F
89502024-01-24T19:09:22.738Z 3 060-079 83 83 83 137 137 137 F F F
89512024-01-24T19:09:22.738Z 4 080-099 72 72 72 118 118 118 F F F
89522024-01-24T19:09:22.738Z 5 100-119 82 82 82 135 135 135 F F F
89532024-01-24T19:09:22.739Z 6 120-139 83 83 83 137 137 137 F F F
89542024-01-24T19:09:22.739Z 7 140-159 83 83 83 137 137 137 F F F
89552024-01-24T19:09:22.739Z 8 160-179 82 82 82 135 135 135 F F F
89562024-01-24T19:09:22.739Z 9 180-199 82 82 82 135 135 135 F F F
89572024-01-24T19:09:22.739Z 10 200-219 83 83 83 137 137 137 F F F
89582024-01-24T19:09:22.739Z 11 220-239 83 83 83 137 137 137 F F F
89592024-01-24T19:09:22.739Z 12 240-259 79 79 79 130 130 130 F F F
89602024-01-24T19:09:22.739Z 13 260-279 79 79 79 130 130 130 F F F
89612024-01-24T19:09:22.739Z 14 280-299 78 78 78 129 129 129 F F F
89622024-01-24T19:09:22.739Z 15 300-319 82 82 82 135 135 135 F F F
89632024-01-24T19:09:22.740Z 16 320-339 82 82 82 135 135 135 F F F
89642024-01-24T19:09:22.740Z 17 340-359 76 76 76 125 125 125 F F F
89652024-01-24T19:09:22.740Z 18 360-379 80 80 80 133 133 133 F F F
89662024-01-24T19:09:22.740Z 19 380-399 83 83 83 137 137 137 F F F
89672024-01-24T19:09:22.740Z 20 400-419 76 76 76 125 125 125 F F F
89682024-01-24T19:09:22.740Z 21 420-439 77 77 77 126 126 126 F F F
89692024-01-24T19:09:22.740Z 22 440-459 78 78 78 127 127 127 F F F
89702024-01-24T19:09:22.740Z 23 460-479 75 75 75 122 122 122 F F F
89712024-01-24T19:09:22.740Z 24 480-499 83 83 83 137 137 137 F F F
89722024-01-24T19:09:22.740Z 25 500-519 83 83 83 137 137 137 F F F
89732024-01-24T19:09:22.741Z 26 520-539 80 80 80 132 132 132 F F F
89742024-01-24T19:09:22.741Z 27 540-559 83 83 83 137 137 137 F F F
89752024-01-24T19:09:22.741Z 28 560-579 81 81 81 134 134 134 F F F
89762024-01-24T19:09:22.742Z 29 580-599 72 72 72 118 118 118 F F F
89772024-01-24T19:09:22.744ZMax gen: 83, Max flush: 137
89782024-01-24T19:09:22.744Z
89792024-01-24T19:09:22.744ZBegin loop 42
89802024-01-24T19:09:22.744ZDownstairs to restart: 2
89812024-01-24T19:09:22.794Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.411873676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465}
89822024-01-24T19:09:22.794Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41233762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465}
89832024-01-24T19:09:22.794Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41241091Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465}
89842024-01-24T19:09:22.794Z{"msg":"Crucible df45611c-9279-4b36-9449-745905f601bf has session id: 1da76d41-b18f-48eb-a825-2a5703cdf302","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.412440903Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89852024-01-24T19:09:22.794Z{"msg":"Upstairs opts: Upstairs UUID: df45611c-9279-4b36-9449-745905f601bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41251312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89862024-01-24T19:09:22.794Z{"msg":"Crucible stats registered with UUID: df45611c-9279-4b36-9449-745905f601bf","v":0,"name":"crucible"Crucible runtime is spawned
89872024-01-24T19:09:22.795Z,"level":30,"time":"2024-01-24T19:09:21.412706891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89882024-01-24T19:09:22.795Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.412787568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465}
89892024-01-24T19:09:22.795Z{"msg":"df45611c-9279-4b36-9449-745905f601bf active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413068736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89902024-01-24T19:09:22.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41318394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89912024-01-24T19:09:22.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413273222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89922024-01-24T19:09:22.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413319754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89932024-01-24T19:09:22.795Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413360265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89942024-01-24T19:09:22.795Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413399782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89952024-01-24T19:09:22.796Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413440418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89962024-01-24T19:09:22.796Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413482394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89972024-01-24T19:09:22.796Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413521022Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89982024-01-24T19:09:22.796Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413613024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
89992024-01-24T19:09:22.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413683601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90002024-01-24T19:09:22.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413766209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90012024-01-24T19:09:22.796Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.413816532Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90022024-01-24T19:09:22.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.415054685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90032024-01-24T19:09:22.797Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.415238087Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90042024-01-24T19:09:22.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.415313949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90052024-01-24T19:09:22.797Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41535184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90062024-01-24T19:09:22.797Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.416130433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90072024-01-24T19:09:22.797Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.416360417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90082024-01-24T19:09:22.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.416576212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90092024-01-24T19:09:22.797Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.416675229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90102024-01-24T19:09:22.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417095916Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90112024-01-24T19:09:22.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417267093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90122024-01-24T19:09:22.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417408739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90132024-01-24T19:09:22.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41746658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90142024-01-24T19:09:22.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417510529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90152024-01-24T19:09:22.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417578988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90162024-01-24T19:09:22.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.417889679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90172024-01-24T19:09:22.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418590309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90182024-01-24T19:09:22.811Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418688946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90192024-01-24T19:09:22.811Z{"msg":"[0]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 137, 137, 135, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418736491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90202024-01-24T19:09:22.812Z{"msg":"[0]R generation[0..12]: [81, 80, 83, 83, 72, 82, 83, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418780425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90212024-01-24T19:09:22.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418823782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90222024-01-24T19:09:22.813Z{"msg":"[1]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 137, 137, 135, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418868094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90232024-01-24T19:09:22.813Z{"msg":"[1]R generation[0..12]: [81, 80, 83, 83, 72, 82, 83, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.418909995Z","hostname":"Wait for a query_work_queue command to finish before sending IO
90242024-01-24T19:09:22.813Zip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90252024-01-24T19:09:22.813Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419016303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90262024-01-24T19:09:22.813Z{"msg":"[2]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 137, 137, 135, 135, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419094688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90272024-01-24T19:09:22.814Z{"msg":"[2]R generation[0..12]: [81, 80, 83, 83, 72, 82, 83, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419141398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90282024-01-24T19:09:22.814Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419198644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90292024-01-24T19:09:22.815Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419240939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90302024-01-24T19:09:22.815Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419307189Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90312024-01-24T19:09:22.815Z{"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419350276Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90322024-01-24T19:09:22.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419419734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90332024-01-24T19:09:22.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419463299Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90342024-01-24T19:09:22.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419517618Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90352024-01-24T19:09:22.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419560382Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90362024-01-24T19:09:22.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.41961754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90372024-01-24T19:09:22.816Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419661686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90382024-01-24T19:09:22.817ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90392024-01-24T19:09:22.817Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419727315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90402024-01-24T19:09:22.818Z{"msg":"df45611c-9279-4b36-9449-745905f601bf is now active with session: 1da76d41-b18f-48eb-a825-2a5703cdf302","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419770903Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90412024-01-24T19:09:22.818Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419823336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465,"session_id":"1da76d41-b18f-48eb-a825-2a5703cdf302"}
90422024-01-24T19:09:22.818Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:21.419863824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1465}
90432024-01-24T19:09:22.818ZLoading write count information from file "/tmp/test_repair_verify.data"
90442024-01-24T19:09:22.818ZRun Repair workload
90452024-01-24T19:09:22.818Z01/30 Write block 311 len 16384 data: 5 5 6 7
90462024-01-24T19:09:22.819Z02/30 Write block 343 len 4096 data: 8
90472024-01-24T19:09:22.819Z03/30 Read block 241 len 16384
90482024-01-24T19:09:22.819Z04/30 Read block 153 len 4096
90492024-01-24T19:09:22.819Z05/30 Write block 313 len 4096 data: 7
90502024-01-24T19:09:22.819Z06/30 Flush
90512024-01-24T19:09:22.819Z07/30 Write block 392 len 4096 data: 6
90522024-01-24T19:09:22.819Z08/30 Read block 410 len 24576
90532024-01-24T19:09:22.819Z09/30 Read block 529 len 4096
90542024-01-24T19:09:22.819Z10/30 Write block 351 len 36864 data: 4 3 3 3 3 5 4 6 7
90552024-01-24T19:09:22.819Z11/30 Flush
90562024-01-24T19:09:22.820Z12/30 Write block 309 len 8192 data: 3 4
90572024-01-24T19:09:22.820Z13/30 Write block 581 len 20480 data: 7 6 7 7 6
90582024-01-24T19:09:22.820Z14/30 Read block 400 len 12288
90592024-01-24T19:09:22.820Z15/30 Read block 316 len 28672
90602024-01-24T19:09:22.820Z16/30 Write block 565 len 24576 data: 6 6 7 7 7 5
90612024-01-24T19:09:22.821Z17/30 Read block 297 len 40960
90622024-01-24T19:09:22.821Z18/30 Read block 266 len 24576
90632024-01-24T19:09:22.821Z19/30 Flush
90642024-01-24T19:09:22.821Z20/30 Write block 440 len 28672 data: 5 4 4 6 6 6 6
90652024-01-24T19:09:22.821Z21/30 Read block 480 len 28672
90662024-01-24T19:09:22.821Z22/30 Write block 131 len 4096 data: 9
90672024-01-24T19:09:22.821Z23/30 Flush
90682024-01-24T19:09:22.821Z24/30 Write block 221 len 40960 data: 7 8 5 6 8 8 8 7 5 6
90692024-01-24T19:09:22.821Z25/30 Read block 75 len 28672
90702024-01-24T19:09:22.821Z26/30 Write block 283 len 24576 data: 10 7 5 4 6 8
90712024-01-24T19:09:22.821Z27/30 Write block 286 len 4096 data: 5
90722024-01-24T19:09:22.821Z28/30 Read block 585 len 36864
90732024-01-24T19:09:22.836Z29/30 Write block 485 len 36864 data: 5 6 4 4 4 5 5 7 8
90742024-01-24T19:09:22.837Z30/30 Write block 416 len 24576 data: 4 5 7 9 7 7
90752024-01-24T19:09:22.837Z----------------------------------------------------------------
90762024-01-24T19:09:22.837Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:24
90772024-01-24T19:09:22.839ZStates: Active Active Active
90782024-01-24T19:09:22.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90792024-01-24T19:09:22.840Z 7 Acked 1006 Write 1 Done Done Sent false
90802024-01-24T19:09:22.840Z 8 Acked 1007 Read 6 Done Done Sent false
90812024-01-24T19:09:22.840Z 9 Acked 1008 Read 1 Done Done Sent false
90822024-01-24T19:09:22.840Z 10 Acked 1009 Write 9 Done Done Sent false
90832024-01-24T19:09:22.840Z 11 Acked 1010 Flush 0 Done Done Sent false
90842024-01-24T19:09:22.841Z 12 Acked 1011 Write 2 Done Done Sent false
90852024-01-24T19:09:22.841Z 13 Acked 1012 Write 5 Done Done Sent false
90862024-01-24T19:09:22.841Z 14 Acked 1013 Read 3 Done Done Sent false
90872024-01-24T19:09:22.841Z 15 Acked 1014 Read 7 Done Done Sent false
90882024-01-24T19:09:22.841Z 16 Acked 1015 Write 6 Done Done Sent false
90892024-01-24T19:09:22.841Z 17 Acked 1016 Read 10 Done Done Sent false
90902024-01-24T19:09:22.842Z 18 Acked 1017 Read 6 Done Done Sent false
90912024-01-24T19:09:22.842Z 19 Acked 1018 Flush 0 Done Done Sent false
90922024-01-24T19:09:22.842Z 20 Acked 1019 Write 7 Done Done Sent false
90932024-01-24T19:09:22.842Z 21 Acked 1020 Read 7 Done Done Sent false
90942024-01-24T19:09:22.842Z 22 Acked 1021 Write 1 Done Done Sent false
90952024-01-24T19:09:22.842Z 23 Acked 1022 Flush 0 Done Done Sent false
90962024-01-24T19:09:22.843Z 24 Acked 1023 Write 10 Done Done Sent false
90972024-01-24T19:09:22.843Z 25 Acked 1024 Read 7 Done Done Sent false
90982024-01-24T19:09:22.843Z 26 Acked 1025 Write 6 Done Done Sent false
90992024-01-24T19:09:22.843Z 27 Acked 1026 Write 1 Done Done Sent false
91002024-01-24T19:09:22.843Z 28 Acked 1027 Read 9 Done Done Sent false
91012024-01-24T19:09:22.844Z 29 Acked 1028 Write 9 Sent Sent Sent false
91022024-01-24T19:09:22.844Z 30 Acked 1029 Write 6 Sent Sent Sent false
91032024-01-24T19:09:22.844Z STATES DS:0 DS:1 DS:2 TOTAL
91042024-01-24T19:09:22.844Z New 0 0 0 0
91052024-01-24T19:09:22.845Z Sent 2 2 24 28
91062024-01-24T19:09:22.845Z Done 22 22 0 44
91072024-01-24T19:09:22.845Z Skipped 0 0 0 0
91082024-01-24T19:09:22.845Z Error 0 0 0 0
91092024-01-24T19:09:22.845ZLast Flush: 1022 1022 1005
91102024-01-24T19:09:22.846ZDownstairs last five completed: 1005 1004 1003 1002 1001
91112024-01-24T19:09:22.847ZUpstairs last five completed: 30 29 28 27 26
91122024-01-24T19:09:22.847ZWrote out file "/tmp/test_repair_verify.data"
91132024-01-24T19:09:22.847Z
91142024-01-24T19:09:22.847ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91152024-01-24T19:09:22.861ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91162024-01-24T19:09:22.861Z 0 000-019 81 81 81 134 134 134 F F F
91172024-01-24T19:09:22.861Z 1 020-039 80 80 80 132 132 132 F F F
91182024-01-24T19:09:22.861Z 2 040-059 83 83 83 137 137 137 F F F
91192024-01-24T19:09:22.861Z 3 060-079 83 83 83 137 137 137 F F F
91202024-01-24T19:09:22.862Z 4 080-099 72 72 72 118 118 118 F F F
91212024-01-24T19:09:22.862Z 5 100-119 82 82 82 135 135 135 F F F
91222024-01-24T19:09:22.862Z 6 120-139 84 84 83 141 141 137 F F F <---
91232024-01-24T19:09:22.862Z 7 140-159 83 83 83 137 137 137 F F F
91242024-01-24T19:09:22.862Z 8 160-179 82 82 82 135 135 135 F F F
91252024-01-24T19:09:22.863Z 9 180-199 82 82 82 135 135 135 F F F
91262024-01-24T19:09:22.863Z 10 200-219 83 83 83 137 137 137 F F F
91272024-01-24T19:09:22.863Z 11 220-239 83 83 83 137 137 137 T T F <---
91282024-01-24T19:09:22.863Z 12 240-259 79 79 79 130 130 130 F F F
91292024-01-24T19:09:22.863Z 13 260-279 79 79 79 130 130 130 F F F
91302024-01-24T19:09:22.863Z 14 280-299 78 78 78 129 129 129 T T F <---
91312024-01-24T19:09:22.863Z 15 300-319 84 84 84 140 140 138 F F F <---
91322024-01-24T19:09:22.863Z 16 320-339 82 82 82 135 135 135 F F F
91332024-01-24T19:09:22.864Z 17 340-359 84 84 84 139 139 138 F F F <---
91342024-01-24T19:09:22.865Z 18 360-379 80 80 80 133 133 133 F F F
91352024-01-24T19:09:22.865Z 19 380-399 84 84 83 139 139 137 F F F <---
91362024-01-24T19:09:22.865Z 20 400-419 76 76 76 125 125 125 T T F <---
91372024-01-24T19:09:22.865Z 21 420-439 77 77 77 126 126 126 T T F <---
91382024-01-24T19:09:22.865Z 22 440-459 84 84 78 141 141 127 F F F <---
91392024-01-24T19:09:22.866Z 23 460-479 75 75 75 122 122 122 F F F
91402024-01-24T19:09:22.866Z 24 480-499 83 83 83 137 137 137 T T F <---
91412024-01-24T19:09:22.866Z 25 500-519 83 83 83 137 137 137 F F F
91422024-01-24T19:09:22.866Z 26 520-539 80 80 80 132 132 132 F F F
91432024-01-24T19:09:22.866Z 27 540-559 83 83 83 137 137 137 F F F
91442024-01-24T19:09:22.866Z 28 560-579 84 84 81 140 140 134 F F F <---
91452024-01-24T19:09:22.866Z 29 580-599 84 84 72 140 140 118 F F F <---
91462024-01-24T19:09:22.866ZMax gen: 84, Max flush: 141
91472024-01-24T19:09:22.866ZError: Difference in extent metadata found!
91482024-01-24T19:09:22.866ZA Difference in extent metadata is expected here
91492024-01-24T19:09:22.866ZOn loop 42
91502024-01-24T19:09:22.867Z
91512024-01-24T19:09:22.879ZVerifying data now
91522024-01-24T19:09:27.946ZNo change after verify
91532024-01-24T19:09:27.946ZLoop: 42 Downstairs dump after verify (and repair):
91542024-01-24T19:09:27.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91552024-01-24T19:09:27.962Z 0 000-019 81 81 81 134 134 134 F F F
91562024-01-24T19:09:27.975Z 1 020-039 80 80 80 132 132 132 F F F
91572024-01-24T19:09:27.975Z 2 040-059 83 83 83 137 137 137 F F F
91582024-01-24T19:09:27.976Z 3 060-079 83 83 83 137 137 137 F F F
91592024-01-24T19:09:27.976Z 4 080-099 72 72 72 118 118 118 F F F
91602024-01-24T19:09:27.976Z 5 100-119 82 82 82 135 135 135 F F F
91612024-01-24T19:09:27.976Z 6 120-139 84 84 84 141 141 141 F F F
91622024-01-24T19:09:27.976Z 7 140-159 83 83 83 137 137 137 F F F
91632024-01-24T19:09:27.976Z 8 160-179 82 82 82 135 135 135 F F F
91642024-01-24T19:09:27.977Z 9 180-199 82 82 82 135 135 135 F F F
91652024-01-24T19:09:27.977Z 10 200-219 83 83 83 137 137 137 F F F
91662024-01-24T19:09:27.977Z 11 220-239 85 85 85 142 142 142 F F F
91672024-01-24T19:09:27.977Z 12 240-259 79 79 79 130 130 130 F F F
91682024-01-24T19:09:27.977Z 13 260-279 79 79 79 130 130 130 F F F
91692024-01-24T19:09:27.977Z 14 280-299 85 85 85 142 142 142 F F F
91702024-01-24T19:09:27.978Z 15 300-319 84 84 84 140 140 140 F F F
91712024-01-24T19:09:27.978Z 16 320-339 82 82 82 135 135 135 F F F
91722024-01-24T19:09:27.978Z 17 340-359 84 84 84 139 139 139 F F F
91732024-01-24T19:09:27.978Z 18 360-379 80 80 80 133 133 133 F F F
91742024-01-24T19:09:27.979Z 19 380-399 84 84 84 139 139 139 F F F
91752024-01-24T19:09:27.979Z 20 400-419 85 85 85 142 142 142 F F F
91762024-01-24T19:09:27.979Z 21 420-439 85 85 85 142 142 142 F F F
91772024-01-24T19:09:27.980Z 22 440-459 84 84 84 141 141 141 F F F
91782024-01-24T19:09:27.980Z 23 460-479 75 75 75 122 122 122 F F F
91792024-01-24T19:09:27.980Z 24 480-499 85 85 85 142 142 142 F F F
91802024-01-24T19:09:27.980Z 25 500-519 83 83 83 137 137 137 F F F
91812024-01-24T19:09:27.980Z 26 520-539 80 80 80 132 132 132 F F F
91822024-01-24T19:09:27.980Z 27 540-559 83 83 83 137 137 137 F F F
91832024-01-24T19:09:27.980Z 28 560-579 84 84 84 140 140 140 F F F
91842024-01-24T19:09:27.980Z 29 580-599 84 84 84 140 140 140 F F F
91852024-01-24T19:09:27.982ZMax gen: 85, Max flush: 142
91862024-01-24T19:09:27.982Z
91872024-01-24T19:09:27.982ZBegin loop 43
91882024-01-24T19:09:27.982ZDownstairs to restart: 1
91892024-01-24T19:09:27.998Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661184993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473}
91902024-01-24T19:09:27.998Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661638285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473}
91912024-01-24T19:09:28.007Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661691962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473}
91922024-01-24T19:09:28.007Z{"msg":"Crucible 4c3e950e-f081-4cfb-8869-73000dc5aafa has session id: 1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661767306Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
91932024-01-24T19:09:28.007Z{"msg":"Upstairs opts: Upstairs UUID: 4c3e950e-f081-4cfb-8869-73000dc5aafa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661870906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
91942024-01-24T19:09:28.007Z{"msg":"Crucible stats registered with UUID: 4c3e950e-f081-4cfb-8869-73000dc5aafa","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.661948246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
91952024-01-24T19:09:28.007ZCrucible runtime is spawned
91962024-01-24T19:09:28.007Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662293747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473}
91972024-01-24T19:09:28.007Z{"msg":"4c3e950e-f081-4cfb-8869-73000dc5aafa active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662457157Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
91982024-01-24T19:09:28.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662526721Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
91992024-01-24T19:09:28.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662574457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92002024-01-24T19:09:28.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662617894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92012024-01-24T19:09:28.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662659969Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92022024-01-24T19:09:28.014Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662734898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92032024-01-24T19:09:28.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662783851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92042024-01-24T19:09:28.014Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662824849Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92052024-01-24T19:09:28.014Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662853303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92062024-01-24T19:09:28.014Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.662917482Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92072024-01-24T19:09:28.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.66319035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92082024-01-24T19:09:28.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.663253578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92092024-01-24T19:09:28.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.663371152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92102024-01-24T19:09:28.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.664527068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92112024-01-24T19:09:28.021Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.664588267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92122024-01-24T19:09:28.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.664840677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92132024-01-24T19:09:28.021Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.664919627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92142024-01-24T19:09:28.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.664953139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92152024-01-24T19:09:28.021Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.665005524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92162024-01-24T19:09:28.021Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.665636245Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92172024-01-24T19:09:28.021Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.66583764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92182024-01-24T19:09:28.021Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666232035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92192024-01-24T19:09:28.021Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666676643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92202024-01-24T19:09:28.021Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666753353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92212024-01-24T19:09:28.021Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666779871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92222024-01-24T19:09:28.021Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666914777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92232024-01-24T19:09:28.021Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.666974796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92242024-01-24T19:09:28.022Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667001442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92252024-01-24T19:09:28.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667195069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92262024-01-24T19:09:28.022Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667274544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92272024-01-24T19:09:28.022Z{"msg":"[0]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 141, 137, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667306612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92282024-01-24T19:09:28.022Z{"msg":"[0]R generation[0..12]: [81, 80, 83, 83, 72, 82, 84, 83, 82, 82, 83, 85]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667354141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92292024-01-24T19:09:28.022Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.66738149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92302024-01-24T19:09:28.022Z{"msg":"[1]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 141, 137, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.66742659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92312024-01-24T19:09:28.022Z{"msg":"[1]R generation[0..12]: [81, 80, 83, 83, 72, 82, 84, 83, 82, 82, 83, 85]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667453024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92322024-01-24T19:09:28.023Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667476981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92332024-01-24T19:09:28.023Z{"msg":"Wait for a query_work_queue command to finish before sending IO
92342024-01-24T19:09:28.023Z[2]R flush_numbers[0..12]: [134, 132, 137, 137, 118, 135, 141, 137, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667527282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92352024-01-24T19:09:28.023Z{"msg":"[2]R generation[0..12]: [81, 80, 83, 83, 72, 82, 84, 83, 82, 82, 83, 85]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667554024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92362024-01-24T19:09:28.023Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667598937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92372024-01-24T19:09:28.023Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.66762452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92382024-01-24T19:09:28.023Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667668253Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92392024-01-24T19:09:28.023Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667692748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92402024-01-24T19:09:28.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667715294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92412024-01-24T19:09:28.024Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667737305Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92422024-01-24T19:09:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667778746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92432024-01-24T19:09:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667804013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92442024-01-24T19:09:28.024Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667825695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92452024-01-24T19:09:28.032Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667847285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92462024-01-24T19:09:28.033Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667894629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92472024-01-24T19:09:28.033Z{"msg":"4c3e950e-f081-4cfb-8869-73000dc5aafa is now active with session: 1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667919879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92482024-01-24T19:09:28.033Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667939697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473,"session_id":"1748a3f7-f7bd-4f98-a112-7ddbdb6dd1b4"}
92492024-01-24T19:09:28.034Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:26.667980692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1473}
92502024-01-24T19:09:28.034ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92512024-01-24T19:09:28.034ZLoading write count information from file "/tmp/test_repair_verify.data"
92522024-01-24T19:09:28.034ZRun Repair workload
92532024-01-24T19:09:28.034Z01/30 Write block 2 len 8192 data: 4 5
92542024-01-24T19:09:28.034Z02/30 Read block 358 len 36864
92552024-01-24T19:09:28.034Z03/30 Read block 311 len 32768
92562024-01-24T19:09:28.034Z04/30 Write block 261 len 12288 data: 6 6 6
92572024-01-24T19:09:28.034Z05/30 Read block 341 len 36864
92582024-01-24T19:09:28.034Z06/30 Read block 173 len 36864
92592024-01-24T19:09:28.034Z07/30 Write block 349 len 4096 data: 4
92602024-01-24T19:09:28.034Z08/30 Write block 44 len 20480 data: 8 10 8 6 8
92612024-01-24T19:09:28.034Z09/30 Write block 456 len 4096 data: 9
92622024-01-24T19:09:28.034Z10/30 Read block 94 len 32768
92632024-01-24T19:09:28.034Z11/30 Read block 568 len 4096
92642024-01-24T19:09:28.034Z12/30 Flush
92652024-01-24T19:09:28.034Z13/30 Write block 137 len 16384 data: 11 9 10 8
92662024-01-24T19:09:28.034Z14/30 Read block 19 len 24576
92672024-01-24T19:09:28.034Z15/30 Read block 53 len 12288
92682024-01-24T19:09:28.034Z16/30 Write block 31 len 12288 data: 11 9 10
92692024-01-24T19:09:28.038Z17/30 Write block 507 len 16384 data: 7 7 7 11
92702024-01-24T19:09:28.038Z18/30 Read block 266 len 20480
92712024-01-24T19:09:28.038Z19/30 Read block 121 len 32768
92722024-01-24T19:09:28.038Z20/30 Read block 400 len 8192
92732024-01-24T19:09:28.038Z21/30 Read block 518 len 32768
92742024-01-24T19:09:28.038Z22/30 Write block 11 len 4096 data: 5
92752024-01-24T19:09:28.038Z23/30 Write block 18 len 20480 data: 4 6 6 7 7
92762024-01-24T19:09:28.038Z24/30 Write block 133 len 8192 data: 6 8
92772024-01-24T19:09:28.039Z25/30 Read block 102 len 4096
92782024-01-24T19:09:28.044Z26/30 Write block 585 len 16384 data: 7 7 7 7
92792024-01-24T19:09:28.045Z27/30 Read block 128 len 24576
92802024-01-24T19:09:28.046Z28/30 Read block 374 len 16384
92812024-01-24T19:09:28.056Z29/30 Write block 429 len 24576 data: 9 8 9 10 10 10
92822024-01-24T19:09:28.056Z30/30 Read block 447 len 24576
92832024-01-24T19:09:28.056Z----------------------------------------------------------------
92842024-01-24T19:09:28.056Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
92852024-01-24T19:09:28.056ZStates: Active Active Active
92862024-01-24T19:09:28.056ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
92872024-01-24T19:09:28.056Z 1 Acked 1000 Write 2 Done Done Done false
92882024-01-24T19:09:28.056Z 2 Acked 1001 Read 9 Done Sent Done false
92892024-01-24T19:09:28.056Z 3 Acked 1002 Read 8 Done Sent Done false
92902024-01-24T19:09:28.056Z 4 Acked 1003 Write 3 Done Sent Done false
92912024-01-24T19:09:28.056Z 5 Acked 1004 Read 9 Done Sent Done false
92922024-01-24T19:09:28.056Z 6 Acked 1005 Read 9 Done Sent Done false
92932024-01-24T19:09:28.076Z 7 Acked 1006 Write 1 Done Sent Done false
92942024-01-24T19:09:28.076Z 8 Acked 1007 Write 5 Done Sent Done false
92952024-01-24T19:09:28.076Z 9 Acked 1008 Write 1 Done Sent Done false
92962024-01-24T19:09:28.076Z 10 Acked 1009 Read 8 Done Sent Done false
92972024-01-24T19:09:28.076Z 11 Acked 1010 Read 1 Done Sent Done false
92982024-01-24T19:09:28.076Z 12 Acked 1011 Flush 0 Done Sent Done false
92992024-01-24T19:09:28.076Z 13 Acked 1012 Write 4 Done Sent Done false
93002024-01-24T19:09:28.076Z 14 Acked 1013 Read 6 Done Sent Done false
93012024-01-24T19:09:28.076Z 15 Acked 1014 Read 3 Done Sent Done false
93022024-01-24T19:09:28.076Z 16 Acked 1015 Write 3 Done Sent Done false
93032024-01-24T19:09:28.076Z 17 Acked 1016 Write 4 Done Sent Done false
93042024-01-24T19:09:28.076Z 18 Acked 1017 Read 5 Done Sent Done false
93052024-01-24T19:09:28.076Z 19 Acked 1018 Read 8 Done Sent Done false
93062024-01-24T19:09:28.076Z 20 Acked 1019 Read 2 Done Sent Done false
93072024-01-24T19:09:28.076Z 21 Acked 1020 Read 8 Done Sent Done false
93082024-01-24T19:09:28.077Z 22 Acked 1021 Write 1 Done Sent Done false
93092024-01-24T19:09:28.077Z 23 Acked 1022 Write 5 Done Sent Done false
93102024-01-24T19:09:28.077Z 24 Acked 1023 Write 2 Done Sent Done false
93112024-01-24T19:09:28.077Z 25 Acked 1024 Read 1 Done Sent Done false
93122024-01-24T19:09:28.077Z 26 Acked 1025 Write 4 Done Sent Done false
93132024-01-24T19:09:28.077Z 27 Acked 1026 Read 6 Done Sent Done false
93142024-01-24T19:09:28.077Z 28 Acked 1027 Read 4 Done Sent Done false
93152024-01-24T19:09:28.077Z 29 Acked 1028 Write 6 Done Sent Done false
93162024-01-24T19:09:28.077Z 30 Acked 1029 Read 6 Done Sent Done false
93172024-01-24T19:09:28.077Z STATES DS:0 DS:1 DS:2 TOTAL
93182024-01-24T19:09:28.078Z New 0 0 0 0
93192024-01-24T19:09:28.078Z Sent 0 29 0 29
93202024-01-24T19:09:28.078Z Done 30 1 30 61
93212024-01-24T19:09:28.078Z Skipped 0 0 0 0
93222024-01-24T19:09:28.078Z Error 0 0 0 0
93232024-01-24T19:09:28.078ZLast Flush: 1011 0 1011
93242024-01-24T19:09:28.078ZDownstairs last five completed:
93252024-01-24T19:09:28.078ZUpstairs last five completed: 30 29 28 27 26
93262024-01-24T19:09:28.078ZWrote out file "/tmp/test_repair_verify.data"
93272024-01-24T19:09:28.082Z
93282024-01-24T19:09:28.082ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93292024-01-24T19:09:28.088ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93302024-01-24T19:09:28.088Z 0 000-019 86 81 86 143 134 143 T T T <---
93312024-01-24T19:09:28.088Z 1 020-039 80 80 80 132 132 132 T F T <---
93322024-01-24T19:09:28.088Z 2 040-059 86 83 86 143 137 143 F F F <---
93332024-01-24T19:09:28.088Z 3 060-079 83 83 83 137 137 137 F F F
93342024-01-24T19:09:28.088Z 4 080-099 72 72 72 118 118 118 F F F
93352024-01-24T19:09:28.088Z 5 100-119 82 82 82 135 135 135 F F F
93362024-01-24T19:09:28.088Z 6 120-139 84 84 84 141 141 141 T F T <---
93372024-01-24T19:09:28.088Z 7 140-159 83 83 83 137 137 137 T F T <---
93382024-01-24T19:09:28.088Z 8 160-179 82 82 82 135 135 135 F F F
93392024-01-24T19:09:28.088Z 9 180-199 82 82 82 135 135 135 F F F
93402024-01-24T19:09:28.088Z 10 200-219 83 83 83 137 137 137 F F F
93412024-01-24T19:09:28.089Z 11 220-239 85 85 85 142 142 142 F F F
93422024-01-24T19:09:28.089Z 12 240-259 79 79 79 130 130 130 F F F
93432024-01-24T19:09:28.089Z 13 260-279 86 79 86 143 130 143 F F F <---
93442024-01-24T19:09:28.089Z 14 280-299 85 85 85 142 142 142 F F F
93452024-01-24T19:09:28.089Z 15 300-319 84 84 84 140 140 140 F F F
93462024-01-24T19:09:28.089Z 16 320-339 82 82 82 135 135 135 F F F
93472024-01-24T19:09:28.089Z 17 340-359 86 84 86 143 139 143 F F F <---
93482024-01-24T19:09:28.089Z 18 360-379 80 80 80 133 133 133 F F F
93492024-01-24T19:09:28.089Z 19 380-399 84 84 84 139 139 139 F F F
93502024-01-24T19:09:28.089Z 20 400-419 85 85 85 142 142 142 F F F
93512024-01-24T19:09:28.089Z 21 420-439 85 85 85 142 142 142 T F T <---
93522024-01-24T19:09:28.090Z 22 440-459 86 84 86 143 141 143 F F F <---
93532024-01-24T19:09:28.090Z 23 460-479 75 75 75 122 122 122 F F F
93542024-01-24T19:09:28.090Z 24 480-499 85 85 85 142 142 142 F F F
93552024-01-24T19:09:28.090Z 25 500-519 83 83 83 137 137 137 T F T <---
93562024-01-24T19:09:28.090Z 26 520-539 80 80 80 132 132 132 F F F
93572024-01-24T19:09:28.090Z 27 540-559 83 83 83 137 137 137 F F F
93582024-01-24T19:09:28.090Z 28 560-579 84 84 84 140 140 140 F F F
93592024-01-24T19:09:28.090Z 29 580-599 84 84 84 140 140 140 T F T <---
93602024-01-24T19:09:28.090ZMax gen: 86, Max flush: 143
93612024-01-24T19:09:28.090ZError: Difference in extent metadata found!
93622024-01-24T19:09:28.094ZA Difference in extent metadata is expected here
93632024-01-24T19:09:28.094ZOn loop 43
93642024-01-24T19:09:28.094Z
93652024-01-24T19:09:28.094ZVerifying data now
93662024-01-24T19:09:33.104ZNo change after verify
93672024-01-24T19:09:33.104ZLoop: 43 Downstairs dump after verify (and repair):
93682024-01-24T19:09:33.124ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93692024-01-24T19:09:33.124Z 0 000-019 87 87 87 144 144 144 F F F
93702024-01-24T19:09:33.124Z 1 020-039 87 87 87 144 144 144 F F F
93712024-01-24T19:09:33.125Z 2 040-059 86 86 86 143 143 143 F F F
93722024-01-24T19:09:33.125Z 3 060-079 83 83 83 137 137 137 F F F
93732024-01-24T19:09:33.125Z 4 080-099 72 72 72 118 118 118 F F F
93742024-01-24T19:09:33.125Z 5 100-119 82 82 82 135 135 135 F F F
93752024-01-24T19:09:33.126Z 6 120-139 87 87 87 144 144 144 F F F
93762024-01-24T19:09:33.126Z 7 140-159 87 87 87 144 144 144 F F F
93772024-01-24T19:09:33.130Z 8 160-179 82 82 82 135 135 135 F F F
93782024-01-24T19:09:33.130Z 9 180-199 82 82 82 135 135 135 F F F
93792024-01-24T19:09:33.130Z 10 200-219 83 83 83 137 137 137 F F F
93802024-01-24T19:09:33.130Z 11 220-239 85 85 85 142 142 142 F F F
93812024-01-24T19:09:33.130Z 12 240-259 79 79 79 130 130 130 F F F
93822024-01-24T19:09:33.131Z 13 260-279 86 86 86 143 143 143 F F F
93832024-01-24T19:09:33.131Z 14 280-299 85 85 85 142 142 142 F F F
93842024-01-24T19:09:33.131Z 15 300-319 84 84 84 140 140 140 F F F
93852024-01-24T19:09:33.131Z 16 320-339 82 82 82 135 135 135 F F F
93862024-01-24T19:09:33.131Z 17 340-359 86 86 86 143 143 143 F F F
93872024-01-24T19:09:33.131Z 18 360-379 80 80 80 133 133 133 F F F
93882024-01-24T19:09:33.131Z 19 380-399 84 84 84 139 139 139 F F F
93892024-01-24T19:09:33.131Z 20 400-419 85 85 85 142 142 142 F F F
93902024-01-24T19:09:33.131Z 21 420-439 87 87 87 144 144 144 F F F
93912024-01-24T19:09:33.131Z 22 440-459 86 86 86 143 143 143 F F F
93922024-01-24T19:09:33.131Z 23 460-479 75 75 75 122 122 122 F F F
93932024-01-24T19:09:33.131Z 24 480-499 85 85 85 142 142 142 F F F
93942024-01-24T19:09:33.132Z 25 500-519 87 87 87 144 144 144 F F F
93952024-01-24T19:09:33.132Z 26 520-539 80 80 80 132 132 132 F F F
93962024-01-24T19:09:33.132Z 27 540-559 83 83 83 137 137 137 F F F
93972024-01-24T19:09:33.132Z 28 560-579 84 84 84 140 140 140 F F F
93982024-01-24T19:09:33.132Z 29 580-599 87 87 87 144 144 144 F F F
93992024-01-24T19:09:33.132ZMax gen: 87, Max flush: 144
94002024-01-24T19:09:33.132Z
94012024-01-24T19:09:33.132ZBegin loop 44
94022024-01-24T19:09:33.132ZDownstairs to restart: 2
94032024-01-24T19:09:33.165Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825045845Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481}
94042024-01-24T19:09:33.165Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825493255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481}
94052024-01-24T19:09:33.187Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825523012Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481}
94062024-01-24T19:09:33.188Z{"msg":"Crucible 6bb22ff8-d224-477a-ae00-5d3459ccd934 has session id: cb451f03-2e0e-4b5e-92a2-20fff284b642","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825581378Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94072024-01-24T19:09:33.188Z{"msg":"Upstairs opts: Upstairs UUID: 6bb22ff8-d224-477a-ae00-5d3459ccd934, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825619754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94082024-01-24T19:09:33.188Z{"msg":"Crucible stats registered with UUID: 6bb22ff8-d224-477a-ae00-5d3459ccd934","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.825656595Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94092024-01-24T19:09:33.188ZCrucible runtime is spawned
94102024-01-24T19:09:33.197Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826016575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481}
94112024-01-24T19:09:33.197Z{"msg":"6bb22ff8-d224-477a-ae00-5d3459ccd934 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826305758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94122024-01-24T19:09:33.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826359739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94132024-01-24T19:09:33.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826387491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94142024-01-24T19:09:33.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.82640994Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94152024-01-24T19:09:33.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826442599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94162024-01-24T19:09:33.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826512554Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94172024-01-24T19:09:33.198Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826587682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94182024-01-24T19:09:33.198Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826751253Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94192024-01-24T19:09:33.198Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826838024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94202024-01-24T19:09:33.198Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.82688722Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94212024-01-24T19:09:33.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.826961109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94222024-01-24T19:09:33.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.827006476Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94232024-01-24T19:09:33.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.827149041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94242024-01-24T19:09:33.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.828607248Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94252024-01-24T19:09:33.199Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.828671173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94262024-01-24T19:09:33.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.828902609Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94272024-01-24T19:09:33.199Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.829045414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94282024-01-24T19:09:33.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.829209202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94292024-01-24T19:09:33.199Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.829280363Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94302024-01-24T19:09:33.199Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.829717745Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94312024-01-24T19:09:33.199Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.829897491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94322024-01-24T19:09:33.200Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.83048951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94332024-01-24T19:09:33.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830667001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94342024-01-24T19:09:33.200Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830730292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94352024-01-24T19:09:33.200Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830757839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94362024-01-24T19:09:33.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830799679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94372024-01-24T19:09:33.201Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830844585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94382024-01-24T19:09:33.201Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.830871043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94392024-01-24T19:09:33.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831212092Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94402024-01-24T19:09:33.201Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831287364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94412024-01-24T19:09:33.202Z{"msg":"[0]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831316667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94422024-01-24T19:09:33.202Z{"msg":"[0]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 82, 82, 83, 85]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.83135053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94432024-01-24T19:09:33.202Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831419832Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94442024-01-24T19:09:33.202Z{"msg":"[1]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831455485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94452024-01-24T19:09:33.203Z{"msg":"[1]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 82, 82, 83, 85]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
94462024-01-24T19:09:33.203Z,"time":"2024-01-24T19:09:31.83151531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94472024-01-24T19:09:33.203Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831582775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94482024-01-24T19:09:33.203Z{"msg":"[2]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 135, 135, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831617033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94492024-01-24T19:09:33.203Z{"msg":"[2]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 82, 82, 83, 85]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831677042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94502024-01-24T19:09:33.203Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831712236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94512024-01-24T19:09:33.204Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831769722Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94522024-01-24T19:09:33.204Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.83180273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94532024-01-24T19:09:33.204Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831871867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94542024-01-24T19:09:33.205Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831899622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94552024-01-24T19:09:33.205Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.831969697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94562024-01-24T19:09:33.205Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832000258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94572024-01-24T19:09:33.205Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832072023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94582024-01-24T19:09:33.205Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832104863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94592024-01-24T19:09:33.205Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832169433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94602024-01-24T19:09:33.206Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832195456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94612024-01-24T19:09:33.206Z{"msg":"6bb22ff8-d224-477a-ae00-5d3459ccd934 is now active with session: cb451f03-2e0e-4b5e-92a2-20fff284b642","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832255298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94622024-01-24T19:09:33.206Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.83228139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481,"session_id":"cb451f03-2e0e-4b5e-92a2-20fff284b642"}
94632024-01-24T19:09:33.206Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:31.832393849Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1481}
94642024-01-24T19:09:33.206ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
94652024-01-24T19:09:33.206ZLoading write count information from file "/tmp/test_repair_verify.data"
94662024-01-24T19:09:33.207ZRun Repair workload
94672024-01-24T19:09:33.207Z01/30 Write block 344 len 40960 data: 8 8 5 4 4 5 4 5 4 4
94682024-01-24T19:09:33.207Z02/30 Read block 166 len 20480
94692024-01-24T19:09:33.232Z03/30 Flush
94702024-01-24T19:09:33.233Z04/30 Write block 515 len 40960 data: 9 7 9 10 10 9 9 9 7 9
94712024-01-24T19:09:33.233Z05/30 Read block 155 len 8192
94722024-01-24T19:09:33.233Z06/30 Read block 1 len 24576
94732024-01-24T19:09:33.233Z07/30 Read block 317 len 8192
94742024-01-24T19:09:33.233Z08/30 Flush
94752024-01-24T19:09:33.233Z09/30 Write block 161 len 12288 data: 11 10 12
94762024-01-24T19:09:33.234Z10/30 Read block 97 len 32768
94772024-01-24T19:09:33.234Z11/30 Write block 329 len 16384 data: 7 6 6 8
94782024-01-24T19:09:33.234Z12/30 Read block 76 len 20480
94792024-01-24T19:09:33.234Z13/30 Read block 177 len 12288
94802024-01-24T19:09:33.234Z14/30 Flush
94812024-01-24T19:09:33.236Z15/30 Write block 315 len 32768 data: 8 10 10 10 10 10 10 9
94822024-01-24T19:09:33.236Z16/30 Read block 304 len 4096
94832024-01-24T19:09:33.236Z17/30 Read block 30 len 24576
94842024-01-24T19:09:33.237Z18/30 Write block 314 len 16384 data: 8 9 11 11
94852024-01-24T19:09:33.237Z19/30 Read block 260 len 40960
94862024-01-24T19:09:33.237Z20/30 Write block 229 len 36864 data: 6 7 7 7 8 8 6 5 4
94872024-01-24T19:09:33.237Z21/30 Read block 152 len 24576
94882024-01-24T19:09:33.237Z22/30 Read block 555 len 40960
94892024-01-24T19:09:33.237Z23/30 Read block 535 len 16384
94902024-01-24T19:09:33.237Z24/30 Read block 453 len 12288
94912024-01-24T19:09:33.238Z25/30 Read block 159 len 24576
94922024-01-24T19:09:33.238Z26/30 Read block 561 len 8192
94932024-01-24T19:09:33.238Z27/30 Write block 533 len 24576 data: 7 6 5 6 7 7
94942024-01-24T19:09:33.238Z28/30 Write block 439 len 16384 data: 6 6 5 5
94952024-01-24T19:09:33.238Z29/30 Read block 386 len 28672
94962024-01-24T19:09:33.238Z30/30 Read block 110 len 24576
94972024-01-24T19:09:33.238Z----------------------------------------------------------------
94982024-01-24T19:09:33.239Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
94992024-01-24T19:09:33.239ZStates: Active Active Active
95002024-01-24T19:09:33.239ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95012024-01-24T19:09:33.239Z 1 Acked 1000 Write 10 Done Done Done false
95022024-01-24T19:09:33.239Z 2 Acked 1001 Read 5 Done Done Done false
95032024-01-24T19:09:33.239Z 3 Acked 1002 Flush 0 Done Done Sent false
95042024-01-24T19:09:33.239Z 4 Acked 1003 Write 10 Done Done Sent false
95052024-01-24T19:09:33.239Z 5 Acked 1004 Read 2 Done Done Sent false
95062024-01-24T19:09:33.240Z 6 Acked 1005 Read 6 Done Done Sent false
95072024-01-24T19:09:33.241Z 7 Acked 1006 Read 2 Done Done Sent false
95082024-01-24T19:09:33.241Z 8 Acked 1007 Flush 0 Done Done Sent false
95092024-01-24T19:09:33.241Z 9 Acked 1008 Write 3 Done Done Sent false
95102024-01-24T19:09:33.242Z 10 Acked 1009 Read 8 Done Done Sent false
95112024-01-24T19:09:33.242Z 11 Acked 1010 Write 4 Done Done Sent false
95122024-01-24T19:09:33.242Z 12 Acked 1011 Read 5 Done Done Sent false
95132024-01-24T19:09:33.242Z 13 Acked 1012 Read 3 Done Done Sent false
95142024-01-24T19:09:33.242Z 14 Acked 1013 Flush 0 Done Done Sent false
95152024-01-24T19:09:33.242Z 15 Acked 1014 Write 8 Done Done Sent false
95162024-01-24T19:09:33.242Z 16 Acked 1015 Read 1 Done Done Sent false
95172024-01-24T19:09:33.242Z 17 Acked 1016 Read 6 Done Done Sent false
95182024-01-24T19:09:33.242Z 18 Acked 1017 Write 4 Done Done Sent false
95192024-01-24T19:09:33.243Z 19 Acked 1018 Read 10 Done Done Sent false
95202024-01-24T19:09:33.243Z 20 Acked 1019 Write 9 Done Done Sent false
95212024-01-24T19:09:33.243Z 21 Acked 1020 Read 6 Done Done Sent false
95222024-01-24T19:09:33.243Z 22 Acked 1021 Read 10 Done Done Sent false
95232024-01-24T19:09:33.243Z 23 Acked 1022 Read 4 Done Done Sent false
95242024-01-24T19:09:33.243Z 24 Acked 1023 Read 3 Done Done Sent false
95252024-01-24T19:09:33.243Z 25 Acked 1024 Read 6 Done Done Sent false
95262024-01-24T19:09:33.244Z 26 Acked 1025 Read 2 Done Done Sent false
95272024-01-24T19:09:33.244Z 27 Acked 1026 Write 6 Done Done Sent false
95282024-01-24T19:09:33.244Z 28 Acked 1027 Write 4 Done Done Sent false
95292024-01-24T19:09:33.244Z 29 Acked 1028 Read 7 Done Done Sent false
95302024-01-24T19:09:33.247Z 30 Acked 1029 Read 6 Done Sent Sent false
95312024-01-24T19:09:33.247Z STATES DS:0 DS:1 DS:2 TOTAL
95322024-01-24T19:09:33.247Z New 0 0 0 0
95332024-01-24T19:09:33.247Z Sent 0 1 28 29
95342024-01-24T19:09:33.248Z Done 30 29 2 61
95352024-01-24T19:09:33.248Z Skipped 0 0 0 0
95362024-01-24T19:09:33.248Z Error 0 0 0 0
95372024-01-24T19:09:33.248ZLast Flush: 1013 1013 0
95382024-01-24T19:09:33.248ZDownstairs last five completed:
95392024-01-24T19:09:33.248ZUpstairs last five completed: 30 29 28 27 26
95402024-01-24T19:09:33.248ZWrote out file "/tmp/test_repair_verify.data"
95412024-01-24T19:09:33.258Z
95422024-01-24T19:09:33.259ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95432024-01-24T19:09:33.263ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95442024-01-24T19:09:33.264Z 0 000-019 87 87 87 144 144 144 F F F
95452024-01-24T19:09:33.264Z 1 020-039 87 87 87 144 144 144 F F F
95462024-01-24T19:09:33.264Z 2 040-059 86 86 86 143 143 143 F F F
95472024-01-24T19:09:33.264Z 3 060-079 83 83 83 137 137 137 F F F
95482024-01-24T19:09:33.264Z 4 080-099 72 72 72 118 118 118 F F F
95492024-01-24T19:09:33.265Z 5 100-119 82 82 82 135 135 135 F F F
95502024-01-24T19:09:33.265Z 6 120-139 87 87 87 144 144 144 F F F
95512024-01-24T19:09:33.265Z 7 140-159 87 87 87 144 144 144 F F F
95522024-01-24T19:09:33.265Z 8 160-179 88 88 82 147 147 135 F F F <---
95532024-01-24T19:09:33.265Z 9 180-199 82 82 82 135 135 135 F F F
95542024-01-24T19:09:33.265Z 10 200-219 83 83 83 137 137 137 F F F
95552024-01-24T19:09:33.266Z 11 220-239 85 85 85 142 142 142 T T F <---
95562024-01-24T19:09:33.266Z 12 240-259 79 79 79 130 130 130 F F F
95572024-01-24T19:09:33.267Z 13 260-279 86 86 86 143 143 143 F F F
95582024-01-24T19:09:33.267Z 14 280-299 85 85 85 142 142 142 F F F
95592024-01-24T19:09:33.267Z 15 300-319 84 84 84 140 140 140 T T F <---
95602024-01-24T19:09:33.267Z 16 320-339 88 88 82 147 147 135 T T F <---
95612024-01-24T19:09:33.267Z 17 340-359 88 88 86 145 145 143 F F T <---
95622024-01-24T19:09:33.267Z 18 360-379 80 80 80 133 133 133 F F F
95632024-01-24T19:09:33.268Z 19 380-399 84 84 84 139 139 139 F F F
95642024-01-24T19:09:33.268Z 20 400-419 85 85 85 142 142 142 F F F
95652024-01-24T19:09:33.268Z 21 420-439 87 87 87 144 144 144 T T F <---
95662024-01-24T19:09:33.268Z 22 440-459 86 86 86 143 143 143 T T F <---
95672024-01-24T19:09:33.268Z 23 460-479 75 75 75 122 122 122 F F F
95682024-01-24T19:09:33.268Z 24 480-499 85 85 85 142 142 142 F F F
95692024-01-24T19:09:33.269Z 25 500-519 88 88 87 146 146 144 F F F <---
95702024-01-24T19:09:33.269Z 26 520-539 88 88 80 146 146 132 T T F <---
95712024-01-24T19:09:33.269Z 27 540-559 83 83 83 137 137 137 F F F
95722024-01-24T19:09:33.269Z 28 560-579 84 84 84 140 140 140 F F F
95732024-01-24T19:09:33.269Z 29 580-599 87 87 87 144 144 144 F F F
95742024-01-24T19:09:33.269ZMax gen: 88, Max flush: 147
95752024-01-24T19:09:33.269ZError: Difference in extent metadata found!
95762024-01-24T19:09:33.275ZA Difference in extent metadata is expected here
95772024-01-24T19:09:33.276ZOn loop 44
95782024-01-24T19:09:33.276Z
95792024-01-24T19:09:33.276ZVerifying data now
95802024-01-24T19:09:38.140ZNo change after verify
95812024-01-24T19:09:38.141ZLoop: 44 Downstairs dump after verify (and repair):
95822024-01-24T19:09:38.163ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95832024-01-24T19:09:38.163Z 0 000-019 87 87 87 144 144 144 F F F
95842024-01-24T19:09:38.171Z 1 020-039 87 87 87 144 144 144 F F F
95852024-01-24T19:09:38.172Z 2 040-059 86 86 86 143 143 143 F F F
95862024-01-24T19:09:38.172Z 3 060-079 83 83 83 137 137 137 F F F
95872024-01-24T19:09:38.172Z 4 080-099 72 72 72 118 118 118 F F F
95882024-01-24T19:09:38.172Z 5 100-119 82 82 82 135 135 135 F F F
95892024-01-24T19:09:38.172Z 6 120-139 87 87 87 144 144 144 F F F
95902024-01-24T19:09:38.172Z 7 140-159 87 87 87 144 144 144 F F F
95912024-01-24T19:09:38.172Z 8 160-179 88 88 88 147 147 147 F F F
95922024-01-24T19:09:38.172Z 9 180-199 82 82 82 135 135 135 F F F
95932024-01-24T19:09:38.172Z 10 200-219 83 83 83 137 137 137 F F F
95942024-01-24T19:09:38.173Z 11 220-239 89 89 89 148 148 148 F F F
95952024-01-24T19:09:38.173Z 12 240-259 79 79 79 130 130 130 F F F
95962024-01-24T19:09:38.173Z 13 260-279 86 86 86 143 143 143 F F F
95972024-01-24T19:09:38.173Z 14 280-299 85 85 85 142 142 142 F F F
95982024-01-24T19:09:38.173Z 15 300-319 89 89 89 148 148 148 F F F
95992024-01-24T19:09:38.173Z 16 320-339 89 89 89 148 148 148 F F F
96002024-01-24T19:09:38.173Z 17 340-359 88 88 88 145 145 145 F F F
96012024-01-24T19:09:38.173Z 18 360-379 80 80 80 133 133 133 F F F
96022024-01-24T19:09:38.173Z 19 380-399 84 84 84 139 139 139 F F F
96032024-01-24T19:09:38.173Z 20 400-419 85 85 85 142 142 142 F F F
96042024-01-24T19:09:38.173Z 21 420-439 89 89 89 148 148 148 F F F
96052024-01-24T19:09:38.173Z 22 440-459 89 89 89 148 148 148 F F F
96062024-01-24T19:09:38.173Z 23 460-479 75 75 75 122 122 122 F F F
96072024-01-24T19:09:38.174Z 24 480-499 85 85 85 142 142 142 F F F
96082024-01-24T19:09:38.174Z 25 500-519 88 88 88 146 146 146 F F F
96092024-01-24T19:09:38.174Z 26 520-539 89 89 89 148 148 148 F F F
96102024-01-24T19:09:38.174Z 27 540-559 83 83 83 137 137 137 F F F
96112024-01-24T19:09:38.174Z 28 560-579 84 84 84 140 140 140 F F F
96122024-01-24T19:09:38.174Z 29 580-599 87 87 87 144 144 144 F F F
96132024-01-24T19:09:38.174ZMax gen: 89, Max flush: 148
96142024-01-24T19:09:38.174Z
96152024-01-24T19:09:38.174ZBegin loop 45
96162024-01-24T19:09:38.175ZDownstairs to restart: 1
96172024-01-24T19:09:38.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.859609425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489}
96182024-01-24T19:09:38.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860030121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489}
96192024-01-24T19:09:38.230Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860082864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489}
96202024-01-24T19:09:38.230Z{"msg":"Crucible 74001bef-f526-4dbf-b7af-6a617ce8e611 has session id: 2d525ec8-3a49-4a42-b5a1-59bf28f782b5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860139551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96212024-01-24T19:09:38.231Z{"msg":"Upstairs opts: Upstairs UUID: 74001bef-f526-4dbf-b7af-6a617ce8e611, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86017911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96222024-01-24T19:09:38.231Z{"msg":"Crucible stats registered with UUID: 74001bef-f526-4dbf-b7af-6a617ce8e611","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860208866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96232024-01-24T19:09:38.232ZCrucible runtime is spawned
96242024-01-24T19:09:38.232Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860609977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489}
96252024-01-24T19:09:38.232Z{"msg":"74001bef-f526-4dbf-b7af-6a617ce8e611 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860764219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96262024-01-24T19:09:38.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860871282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96272024-01-24T19:09:38.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860953586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96282024-01-24T19:09:38.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.860999696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96292024-01-24T19:09:38.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861077744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96302024-01-24T19:09:38.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861121975Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96312024-01-24T19:09:38.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861317229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96322024-01-24T19:09:38.235Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86148245Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96332024-01-24T19:09:38.235Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861570112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96342024-01-24T19:09:38.235Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861668442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96352024-01-24T19:09:38.235Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861747247Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96362024-01-24T19:09:38.236Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861828459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96372024-01-24T19:09:38.236Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.861912657Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96382024-01-24T19:09:38.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.862776347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96392024-01-24T19:09:38.236Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86285991Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96402024-01-24T19:09:38.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.862903082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96412024-01-24T19:09:38.236Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.8631675Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96422024-01-24T19:09:38.237Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.863594479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96432024-01-24T19:09:38.237Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.863709999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96442024-01-24T19:09:38.237Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.864040916Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96452024-01-24T19:09:38.237Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.864107533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96462024-01-24T19:09:38.238Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.864695126Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96472024-01-24T19:09:38.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865020849Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96482024-01-24T19:09:38.238Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865082338Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96492024-01-24T19:09:38.238Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865109746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96502024-01-24T19:09:38.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865188967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96512024-01-24T19:09:38.239Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86524339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96522024-01-24T19:09:38.239Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865270496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96532024-01-24T19:09:38.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865580734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96542024-01-24T19:09:38.239Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865636955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96552024-01-24T19:09:38.239Z{"msg":"[0]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865664718Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96562024-01-24T19:09:38.239Z{"msg":"[0]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865713219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96572024-01-24T19:09:38.240Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865740232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96582024-01-24T19:09:38.240Z{"msg":"[1]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865784522Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96592024-01-24T19:09:38.240Z{"msg":"[1]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865810873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96602024-01-24T19:09:38.240Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865855292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96612024-01-24T19:09:38.240Z{"msg":"[2]R flush_numbers[0..12]: [144, 144, 143, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.865882004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96622024-01-24T19:09:38.241Z{"msg":"[2]R generation[0..12]: [87, 87, 86, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
96632024-01-24T19:09:38.241Z,"time":"2024-01-24T19:09:36.865939651Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96642024-01-24T19:09:38.241Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866036458Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96652024-01-24T19:09:38.241Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866099831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96662024-01-24T19:09:38.241Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866197688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96672024-01-24T19:09:38.242Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866241984Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96682024-01-24T19:09:38.242Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866310214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96692024-01-24T19:09:38.242Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866338684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96702024-01-24T19:09:38.242Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866381135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96712024-01-24T19:09:38.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866419698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96722024-01-24T19:09:38.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866480932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96732024-01-24T19:09:38.244Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866520223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96742024-01-24T19:09:38.244Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86658637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96752024-01-24T19:09:38.244Z{"msg":"74001bef-f526-4dbf-b7af-6a617ce8e611 is now active with session: 2d525ec8-3a49-4a42-b5a1-59bf28f782b5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866643418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96762024-01-24T19:09:38.245Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.866667592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489,"session_id":"2d525ec8-3a49-4a42-b5a1-59bf28f782b5"}
96772024-01-24T19:09:38.245Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:36.86669148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1489}
96782024-01-24T19:09:38.245ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96792024-01-24T19:09:38.245ZLoading write count information from file "/tmp/test_repair_verify.data"
96802024-01-24T19:09:38.245ZRun Repair workload
96812024-01-24T19:09:38.245Z01/30 Write block 592 len 12288 data: 10 9 8
96822024-01-24T19:09:38.245Z02/30 Read block 54 len 12288
96832024-01-24T19:09:38.247Z03/30 Flush
96842024-01-24T19:09:38.247Z04/30 Write block 23 len 40960 data: 6 8 8 9 10 11 10 10 12 10
96852024-01-24T19:09:38.247Z05/30 Read block 224 len 36864
96862024-01-24T19:09:38.280Z06/30 Read block 343 len 16384
96872024-01-24T19:09:38.280Z07/30 Read block 381 len 40960
96882024-01-24T19:09:38.281Z08/30 Read block 299 len 8192
96892024-01-24T19:09:38.281Z09/30 Read block 310 len 32768
96902024-01-24T19:09:38.281Z10/30 Flush
96912024-01-24T19:09:38.281Z11/30 Write block 54 len 4096 data: 7
96922024-01-24T19:09:38.282Z12/30 Read block 474 len 8192
96932024-01-24T19:09:38.282Z13/30 Write block 6 len 12288 data: 7 8 9
96942024-01-24T19:09:38.282Z14/30 Write block 48 len 32768 data: 9 8 11 8 7 7 8 8
96952024-01-24T19:09:38.282Z15/30 Read block 572 len 40960
96962024-01-24T19:09:38.282Z16/30 Read block 359 len 28672
96972024-01-24T19:09:38.283Z17/30 Flush
96982024-01-24T19:09:38.283Z18/30 Flush
96992024-01-24T19:09:38.283Z19/30 Write block 41 len 20480 data: 4 4 5 9 11
97002024-01-24T19:09:38.283Z20/30 Flush
97012024-01-24T19:09:38.283Z21/30 Write block 274 len 20480 data: 9 8 6 9 10
97022024-01-24T19:09:38.284Z22/30 Read block 140 len 40960
97032024-01-24T19:09:38.284Z23/30 Read block 497 len 24576
97042024-01-24T19:09:38.284Z24/30 Read block 98 len 28672
97052024-01-24T19:09:38.284Z25/30 Read block 429 len 4096
97062024-01-24T19:09:38.285Z26/30 Read block 509 len 36864
97072024-01-24T19:09:38.285Z27/30 Write block 32 len 40960 data: 11 11 10 5 5 4 4 3 4 5
97082024-01-24T19:09:38.285Z28/30 Write block 432 len 24576 data: 11 11 11 8 7 8
97092024-01-24T19:09:38.285Z29/30 Read block 334 len 4096
97102024-01-24T19:09:38.286Z30/30 Write block 247 len 28672 data: 10 9 10 7 6 4 4
97112024-01-24T19:09:38.286Z----------------------------------------------------------------
97122024-01-24T19:09:38.286Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:27
97132024-01-24T19:09:38.286ZStates: Active Active Active
97142024-01-24T19:09:38.286ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97152024-01-24T19:09:38.287Z 4 Acked 1003 Write 10 Done Sent Done false
97162024-01-24T19:09:38.287Z 5 Acked 1004 Read 9 Done Sent Done false
97172024-01-24T19:09:38.287Z 6 Acked 1005 Read 4 Done Sent Done false
97182024-01-24T19:09:38.288Z 7 Acked 1006 Read 10 Done Sent Done false
97192024-01-24T19:09:38.288Z 8 Acked 1007 Read 2 Done Sent Done false
97202024-01-24T19:09:38.288Z 9 Acked 1008 Read 8 Done Sent Done false
97212024-01-24T19:09:38.288Z 10 Acked 1009 Flush 0 Done Sent Done false
97222024-01-24T19:09:38.288Z 11 Acked 1010 Write 1 Done Sent Done false
97232024-01-24T19:09:38.288Z 12 Acked 1011 Read 2 Done Sent Done false
97242024-01-24T19:09:38.288Z 13 Acked 1012 Write 3 Done Sent Done false
97252024-01-24T19:09:38.288Z 14 Acked 1013 Write 8 Done Sent Done false
97262024-01-24T19:09:38.288Z 15 Acked 1014 Read 10 Done Sent Done false
97272024-01-24T19:09:38.289Z 16 Acked 1015 Read 7 Done Sent Done false
97282024-01-24T19:09:38.289Z 17 Acked 1016 Flush 0 Done Sent Done false
97292024-01-24T19:09:38.289Z 18 Acked 1017 Flush 0 Done Sent Done false
97302024-01-24T19:09:38.289Z 19 Acked 1018 Write 5 Done Sent Done false
97312024-01-24T19:09:38.290Z 20 Acked 1019 Flush 0 Done Sent Done false
97322024-01-24T19:09:38.290Z 21 Acked 1020 Write 5 Done Sent Done false
97332024-01-24T19:09:38.290Z 22 Acked 1021 Read 10 Done Sent Done false
97342024-01-24T19:09:38.290Z 23 Acked 1022 Read 6 Done Sent Done false
97352024-01-24T19:09:38.290Z 24 Acked 1023 Read 7 Done Sent Done false
97362024-01-24T19:09:38.291Z 25 Acked 1024 Read 1 Done Sent Done false
97372024-01-24T19:09:38.291Z 26 Acked 1025 Read 9 Done Sent Done false
97382024-01-24T19:09:38.291Z 27 Acked 1026 Write 10 Done Sent Done false
97392024-01-24T19:09:38.291Z 28 Acked 1027 Write 6 Done Sent Done false
97402024-01-24T19:09:38.291Z 29 Acked 1028 Read 1 Done Sent Done false
97412024-01-24T19:09:38.291Z 30 Acked 1029 Write 7 Sent Sent Sent false
97422024-01-24T19:09:38.291Z STATES DS:0 DS:1 DS:2 TOTAL
97432024-01-24T19:09:38.291Z New 0 0 0 0
97442024-01-24T19:09:38.291Z Sent 1 27 1 29
97452024-01-24T19:09:38.291Z Done 26 0 26 52
97462024-01-24T19:09:38.291Z Skipped 0 0 0 0
97472024-01-24T19:09:38.291Z Error 0 0 0 0
97482024-01-24T19:09:38.291ZLast Flush: 1019 1002 1019
97492024-01-24T19:09:38.292ZDownstairs last five completed: 1002 1001 1000
97502024-01-24T19:09:38.292ZUpstairs last five completed: 30 29 28 27 26
97512024-01-24T19:09:38.292ZWrote out file "/tmp/test_repair_verify.data"
97522024-01-24T19:09:38.307Z
97532024-01-24T19:09:38.307ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97542024-01-24T19:09:38.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97552024-01-24T19:09:38.307Z 0 000-019 90 87 90 151 144 151 F F F <---
97562024-01-24T19:09:38.308Z 1 020-039 90 87 90 150 144 150 T F T <---
97572024-01-24T19:09:38.308Z 2 040-059 90 86 90 153 143 153 T F T <---
97582024-01-24T19:09:38.308Z 3 060-079 83 83 83 137 137 137 F F F
97592024-01-24T19:09:38.308Z 4 080-099 72 72 72 118 118 118 F F F
97602024-01-24T19:09:38.310Z 5 100-119 82 82 82 135 135 135 F F F
97612024-01-24T19:09:38.310Z 6 120-139 87 87 87 144 144 144 F F F
97622024-01-24T19:09:38.310Z 7 140-159 87 87 87 144 144 144 F F F
97632024-01-24T19:09:38.311Z 8 160-179 88 88 88 147 147 147 F F F
97642024-01-24T19:09:38.311Z 9 180-199 82 82 82 135 135 135 F F F
97652024-01-24T19:09:38.311Z 10 200-219 83 83 83 137 137 137 F F F
97662024-01-24T19:09:38.311Z 11 220-239 89 89 89 148 148 148 F F F
97672024-01-24T19:09:38.311Z 12 240-259 79 79 79 130 130 130 T F T <---
97682024-01-24T19:09:38.311Z 13 260-279 86 86 86 143 143 143 T F T <---
97692024-01-24T19:09:38.312Z 14 280-299 85 85 85 142 142 142 F F F
97702024-01-24T19:09:38.312Z 15 300-319 89 89 89 148 148 148 F F F
97712024-01-24T19:09:38.312Z 16 320-339 89 89 89 148 148 148 F F F
97722024-01-24T19:09:38.312Z 17 340-359 88 88 88 145 145 145 F F F
97732024-01-24T19:09:38.312Z 18 360-379 80 80 80 133 133 133 F F F
97742024-01-24T19:09:38.313Z 19 380-399 84 84 84 139 139 139 F F F
97752024-01-24T19:09:38.313Z 20 400-419 85 85 85 142 142 142 F F F
97762024-01-24T19:09:38.313Z 21 420-439 89 89 89 148 148 148 T F T <---
97772024-01-24T19:09:38.313Z 22 440-459 89 89 89 148 148 148 F F F
97782024-01-24T19:09:38.313Z 23 460-479 75 75 75 122 122 122 F F F
97792024-01-24T19:09:38.313Z 24 480-499 85 85 85 142 142 142 F F F
97802024-01-24T19:09:38.314Z 25 500-519 88 88 88 146 146 146 F F F
97812024-01-24T19:09:38.314Z 26 520-539 89 89 89 148 148 148 F F F
97822024-01-24T19:09:38.314Z 27 540-559 83 83 83 137 137 137 F F F
97832024-01-24T19:09:38.314Z 28 560-579 84 84 84 140 140 140 F F F
97842024-01-24T19:09:38.315Z 29 580-599 90 90 90 149 149 149 F F F
97852024-01-24T19:09:38.315ZMax gen: 90, Max flush: 153
97862024-01-24T19:09:38.316ZError: Difference in extent metadata found!
97872024-01-24T19:09:38.326ZA Difference in extent metadata is expected here
97882024-01-24T19:09:38.326ZOn loop 45
97892024-01-24T19:09:38.326Z
97902024-01-24T19:09:38.326ZVerifying data now
97912024-01-24T19:09:42.942ZNo change after verify
97922024-01-24T19:09:42.942ZLoop: 45 Downstairs dump after verify (and repair):
97932024-01-24T19:09:42.965ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97942024-01-24T19:09:42.965Z 0 000-019 90 90 90 151 151 151 F F F
97952024-01-24T19:09:42.970Z 1 020-039 91 91 91 154 154 154 F F F
97962024-01-24T19:09:42.970Z 2 040-059 91 91 91 154 154 154 F F F
97972024-01-24T19:09:42.970Z 3 060-079 83 83 83 137 137 137 F F F
97982024-01-24T19:09:42.970Z 4 080-099 72 72 72 118 118 118 F F F
97992024-01-24T19:09:42.970Z 5 100-119 82 82 82 135 135 135 F F F
98002024-01-24T19:09:42.970Z 6 120-139 87 87 87 144 144 144 F F F
98012024-01-24T19:09:42.970Z 7 140-159 87 87 87 144 144 144 F F F
98022024-01-24T19:09:42.971Z 8 160-179 88 88 88 147 147 147 F F F
98032024-01-24T19:09:42.971Z 9 180-199 82 82 82 135 135 135 F F F
98042024-01-24T19:09:42.971Z 10 200-219 83 83 83 137 137 137 F F F
98052024-01-24T19:09:42.971Z 11 220-239 89 89 89 148 148 148 F F F
98062024-01-24T19:09:42.971Z 12 240-259 91 91 91 154 154 154 F F F
98072024-01-24T19:09:42.971Z 13 260-279 91 91 91 154 154 154 F F F
98082024-01-24T19:09:42.971Z 14 280-299 85 85 85 142 142 142 F F F
98092024-01-24T19:09:42.971Z 15 300-319 89 89 89 148 148 148 F F F
98102024-01-24T19:09:42.971Z 16 320-339 89 89 89 148 148 148 F F F
98112024-01-24T19:09:42.971Z 17 340-359 88 88 88 145 145 145 F F F
98122024-01-24T19:09:42.971Z 18 360-379 80 80 80 133 133 133 F F F
98132024-01-24T19:09:42.971Z 19 380-399 84 84 84 139 139 139 F F F
98142024-01-24T19:09:42.971Z 20 400-419 85 85 85 142 142 142 F F F
98152024-01-24T19:09:42.971Z 21 420-439 91 91 91 154 154 154 F F F
98162024-01-24T19:09:42.971Z 22 440-459 89 89 89 148 148 148 F F F
98172024-01-24T19:09:42.971Z 23 460-479 75 75 75 122 122 122 F F F
98182024-01-24T19:09:42.971Z 24 480-499 85 85 85 142 142 142 F F F
98192024-01-24T19:09:42.971Z 25 500-519 88 88 88 146 146 146 F F F
98202024-01-24T19:09:42.971Z 26 520-539 89 89 89 148 148 148 F F F
98212024-01-24T19:09:42.971Z 27 540-559 83 83 83 137 137 137 F F F
98222024-01-24T19:09:42.971Z 28 560-579 84 84 84 140 140 140 F F F
98232024-01-24T19:09:42.971Z 29 580-599 90 90 90 149 149 149 F F F
98242024-01-24T19:09:42.971ZMax gen: 91, Max flush: 154
98252024-01-24T19:09:42.975Z
98262024-01-24T19:09:42.975ZBegin loop 46
98272024-01-24T19:09:42.975ZDownstairs to restart: 2
98282024-01-24T19:09:43.004Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.66577547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497}
98292024-01-24T19:09:43.004Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666271997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497}
98302024-01-24T19:09:43.004Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666325148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497}
98312024-01-24T19:09:43.027Z{"msg":"Crucible 3371efa9-8918-493f-ae84-5b3c264c443e has session id: 2fd47476-a9ce-42cf-9f17-6914baaa7bb5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666352224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98322024-01-24T19:09:43.027Z{"msg":"Upstairs opts: Upstairs UUID: 3371efa9-8918-493f-ae84-5b3c264c443e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666449255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98332024-01-24T19:09:43.027Z{"msg":"Crucible stats registered with UUID: 3371efa9-8918-493f-ae84-5b3c264c443e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666496797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98342024-01-24T19:09:43.028ZCrucible runtime is spawned
98352024-01-24T19:09:43.028Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666723697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497}
98362024-01-24T19:09:43.028Z{"msg":"3371efa9-8918-493f-ae84-5b3c264c443e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666901646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98372024-01-24T19:09:43.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666966135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98382024-01-24T19:09:43.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.666995513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98392024-01-24T19:09:43.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667053268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98402024-01-24T19:09:43.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.66709898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98412024-01-24T19:09:43.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667140414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98422024-01-24T19:09:43.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667183973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98432024-01-24T19:09:43.028Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667257657Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98442024-01-24T19:09:43.028Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667346767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98452024-01-24T19:09:43.029Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667433004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98462024-01-24T19:09:43.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667564715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98472024-01-24T19:09:43.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667624587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98482024-01-24T19:09:43.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.667657023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98492024-01-24T19:09:43.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669181899Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98502024-01-24T19:09:43.029Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669244955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98512024-01-24T19:09:43.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669429897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98522024-01-24T19:09:43.029Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669493133Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98532024-01-24T19:09:43.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669727679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98542024-01-24T19:09:43.029Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.669781023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98552024-01-24T19:09:43.029Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.670267043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98562024-01-24T19:09:43.031Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.67053871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98572024-01-24T19:09:43.031Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.670826297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98582024-01-24T19:09:43.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671132015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98592024-01-24T19:09:43.031Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671226724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98602024-01-24T19:09:43.031Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671374754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98612024-01-24T19:09:43.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671417163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98622024-01-24T19:09:43.032Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671491131Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98632024-01-24T19:09:43.032Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671566782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98642024-01-24T19:09:43.032Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.671847698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98652024-01-24T19:09:43.032Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.67191666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98662024-01-24T19:09:43.032Z{"msg":"[0]R flush_numbers[0..12]: [151, 154, 154, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672011125Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98672024-01-24T19:09:43.032Z{"msg":"[0]R generation[0..12]: [90, 91, 91, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672073085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98682024-01-24T19:09:43.032Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672101868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98692024-01-24T19:09:43.032Z{"msg":"[1]R flush_numbers[0..12]: [151, 154, 154, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672146666ZWait for a query_work_queue command to finish before sending IO
98702024-01-24T19:09:43.032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98712024-01-24T19:09:43.032Z{"msg":"[1]R generation[0..12]: [90, 91, 91, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672179084Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98722024-01-24T19:09:43.032Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672224328Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98732024-01-24T19:09:43.032Z{"msg":"[2]R flush_numbers[0..12]: [151, 154, 154, 137, 118, 135, 144, 144, 147, 135, 137, 148]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672251166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98742024-01-24T19:09:43.032Z{"msg":"[2]R generation[0..12]: [90, 91, 91, 83, 72, 82, 87, 87, 88, 82, 83, 89]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.67229483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98752024-01-24T19:09:43.032Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672321972Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98762024-01-24T19:09:43.033Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672368074Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98772024-01-24T19:09:43.033Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.67239388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98782024-01-24T19:09:43.033Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672436398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98792024-01-24T19:09:43.033Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672461844Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98802024-01-24T19:09:43.034Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672505621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98812024-01-24T19:09:43.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672529035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98822024-01-24T19:09:43.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672583285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98832024-01-24T19:09:43.034Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672626585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98842024-01-24T19:09:43.034Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672661614Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98852024-01-24T19:09:43.035Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672707843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98862024-01-24T19:09:43.035Z{"msg":"3371efa9-8918-493f-ae84-5b3c264c443e is now active with session: 2fd47476-a9ce-42cf-9f17-6914baaa7bb5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672733722Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98872024-01-24T19:09:43.036Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672775214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497,"session_id":"2fd47476-a9ce-42cf-9f17-6914baaa7bb5"}
98882024-01-24T19:09:43.036Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:41.672799095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1497}
98892024-01-24T19:09:43.036ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98902024-01-24T19:09:43.036ZLoading write count information from file "/tmp/test_repair_verify.data"
98912024-01-24T19:09:43.036ZRun Repair workload
98922024-01-24T19:09:43.036Z01/30 Write block 437 len 16384 data: 9 6 7 7
98932024-01-24T19:09:43.036Z02/30 Read block 459 len 40960
98942024-01-24T19:09:43.113Z03/30 Write block 187 len 4096 data: 7
98952024-01-24T19:09:43.113Z04/30 Read block 533 len 32768
98962024-01-24T19:09:43.114Z05/30 Read block 32 len 40960
98972024-01-24T19:09:43.114Z06/30 Read block 105 len 16384
98982024-01-24T19:09:43.114Z07/30 Write block 567 len 12288 data: 8 8 8
98992024-01-24T19:09:43.114Z08/30 Read block 417 len 24576
99002024-01-24T19:09:43.114Z09/30 Write block 83 len 8192 data: 13 12
99012024-01-24T19:09:43.114Z10/30 Flush
99022024-01-24T19:09:43.114Z11/30 Flush
99032024-01-24T19:09:43.118Z12/30 Write block 554 len 20480 data: 4 3 5 7 7
99042024-01-24T19:09:43.118Z13/30 Read block 327 len 20480
99052024-01-24T19:09:43.118Z14/30 Read block 480 len 4096
99062024-01-24T19:09:43.118Z15/30 Write block 234 len 4096 data: 9
99072024-01-24T19:09:43.118Z16/30 Write block 393 len 40960 data: 9 10 9 10 7 7 6 6 7 6
99082024-01-24T19:09:43.118Z17/30 Write block 544 len 12288 data: 10 9 8
99092024-01-24T19:09:43.118Z18/30 Write block 8 len 32768 data: 10 7 7 6 5 5 4 6
99102024-01-24T19:09:43.118Z19/30 Read block 440 len 24576
99112024-01-24T19:09:43.119Z20/30 Flush
99122024-01-24T19:09:43.119Z21/30 Write block 526 len 4096 data: 9
99132024-01-24T19:09:43.119Z22/30 Flush
99142024-01-24T19:09:43.141Z23/30 Write block 533 len 16384 data: 8 7 6 7
99152024-01-24T19:09:43.141Z24/30 Write block 289 len 40960 data: 7 7 7 7 9 10 9 8 6 6
99162024-01-24T19:09:43.141Z25/30 Write block 279 len 4096 data: 12
99172024-01-24T19:09:43.141Z26/30 Flush
99182024-01-24T19:09:43.141Z27/30 Write block 496 len 4096 data: 10
99192024-01-24T19:09:43.141Z28/30 Write block 349 len 40960 data: 6 5 6 5 5 4 4 6 5 7
99202024-01-24T19:09:43.141Z29/30 Write block 525 len 28672 data: 9 10 9 11 10 9 7
99212024-01-24T19:09:43.141Z30/30 Read block 202 len 36864
99222024-01-24T19:09:43.142Z----------------------------------------------------------------
99232024-01-24T19:09:43.142Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99242024-01-24T19:09:43.142ZStates: Active Active Active
99252024-01-24T19:09:43.142ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99262024-01-24T19:09:43.142Z 1 Acked 1000 Write 4 Done Done Done false
99272024-01-24T19:09:43.142Z 2 Acked 1001 Read 10 Done Done Done false
99282024-01-24T19:09:43.142Z 3 Acked 1002 Write 1 Done Done Sent false
99292024-01-24T19:09:43.142Z 4 Acked 1003 Read 8 Done Done Sent false
99302024-01-24T19:09:43.142Z 5 Acked 1004 Read 10 Done Done Sent false
99312024-01-24T19:09:43.142Z 6 Acked 1005 Read 4 Done Done Sent false
99322024-01-24T19:09:43.142Z 7 Acked 1006 Write 3 Done Done Sent false
99332024-01-24T19:09:43.142Z 8 Acked 1007 Read 6 Done Done Sent false
99342024-01-24T19:09:43.142Z 9 Acked 1008 Write 2 Done Done Sent false
99352024-01-24T19:09:43.143Z 10 Acked 1009 Flush 0 Done Done Sent false
99362024-01-24T19:09:43.143Z 11 Acked 1010 Flush 0 Done Done Sent false
99372024-01-24T19:09:43.143Z 12 Acked 1011 Write 5 Done Done Sent false
99382024-01-24T19:09:43.143Z 13 Acked 1012 Read 5 Done Done Sent false
99392024-01-24T19:09:43.143Z 14 Acked 1013 Read 1 Done Done Sent false
99402024-01-24T19:09:43.143Z 15 Acked 1014 Write 1 Done Done Sent false
99412024-01-24T19:09:43.143Z 16 Acked 1015 Write 10 Done Done Sent false
99422024-01-24T19:09:43.143Z 17 Acked 1016 Write 3 Done Done Sent false
99432024-01-24T19:09:43.143Z 18 Acked 1017 Write 8 Done Done Sent false
99442024-01-24T19:09:43.143Z 19 Acked 1018 Read 6 Done Done Sent false
99452024-01-24T19:09:43.143Z 20 Acked 1019 Flush 0 Done Done Sent false
99462024-01-24T19:09:43.143Z 21 Acked 1020 Write 1 Done Done Sent false
99472024-01-24T19:09:43.143Z 22 Acked 1021 Flush 0 Done Done Sent false
99482024-01-24T19:09:43.143Z 23 Acked 1022 Write 4 Done Done Sent false
99492024-01-24T19:09:43.144Z 24 Acked 1023 Write 10 Done Done Sent false
99502024-01-24T19:09:43.144Z 25 Acked 1024 Write 1 Done Done Sent false
99512024-01-24T19:09:43.145Z 26 Acked 1025 Flush 0 Done Done Sent false
99522024-01-24T19:09:43.145Z 27 Acked 1026 Write 1 Done Done Sent false
99532024-01-24T19:09:43.145Z 28 Acked 1027 Write 10 Done Done Sent false
99542024-01-24T19:09:43.146Z 29 Acked 1028 Write 7 Done Done Sent false
99552024-01-24T19:09:43.146Z 30 Acked 1029 Read 9 Done Done Sent false
99562024-01-24T19:09:43.146Z STATES DS:0 DS:1 DS:2 TOTAL
99572024-01-24T19:09:43.146Z New 0 0 0 0
99582024-01-24T19:09:43.146Z Sent 0 0 28 28
99592024-01-24T19:09:43.146Z Done 30 30 2 62
99602024-01-24T19:09:43.146Z Skipped 0 0 0 0
99612024-01-24T19:09:43.146Z Error 0 0 0 0
99622024-01-24T19:09:43.146ZLast Flush: 1025 1025 0
99632024-01-24T19:09:43.146ZDownstairs last five completed:
99642024-01-24T19:09:43.146ZUpstairs last five completed: 30 29 28 27 26
99652024-01-24T19:09:43.146ZWrote out file "/tmp/test_repair_verify.data"
99662024-01-24T19:09:43.146Z
99672024-01-24T19:09:43.146ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99682024-01-24T19:09:43.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99692024-01-24T19:09:43.147Z 0 000-019 92 92 90 157 157 151 F F F <---
99702024-01-24T19:09:43.147Z 1 020-039 91 91 91 154 154 154 F F F
99712024-01-24T19:09:43.147Z 2 040-059 91 91 91 154 154 154 F F F
99722024-01-24T19:09:43.147Z 3 060-079 83 83 83 137 137 137 F F F
99732024-01-24T19:09:43.147Z 4 080-099 92 92 72 155 155 118 F F F <---
99742024-01-24T19:09:43.147Z 5 100-119 82 82 82 135 135 135 F F F
99752024-01-24T19:09:43.147Z 6 120-139 87 87 87 144 144 144 F F F
99762024-01-24T19:09:43.148Z 7 140-159 87 87 87 144 144 144 F F F
99772024-01-24T19:09:43.148Z 8 160-179 88 88 88 147 147 147 F F F
99782024-01-24T19:09:43.148Z 9 180-199 92 92 82 155 155 135 F F F <---
99792024-01-24T19:09:43.148Z 10 200-219 83 83 83 137 137 137 F F F
99802024-01-24T19:09:43.148Z 11 220-239 92 92 89 157 157 148 F F F <---
99812024-01-24T19:09:43.148Z 12 240-259 91 91 91 154 154 154 F F F
99822024-01-24T19:09:43.148Z 13 260-279 92 92 91 159 159 154 F F F <---
99832024-01-24T19:09:43.148Z 14 280-299 92 92 85 159 159 142 F F F <---
99842024-01-24T19:09:43.148Z 15 300-319 89 89 89 148 148 148 F F F
99852024-01-24T19:09:43.149Z 16 320-339 89 89 89 148 148 148 F F F
99862024-01-24T19:09:43.149Z 17 340-359 88 88 88 145 145 145 T T F <---
99872024-01-24T19:09:43.149Z 18 360-379 80 80 80 133 133 133 F F F
99882024-01-24T19:09:43.149Z 19 380-399 92 92 84 157 157 139 F F F <---
99892024-01-24T19:09:43.149Z 20 400-419 92 92 85 157 157 142 F F F <---
99902024-01-24T19:09:43.149Z 21 420-439 92 92 91 155 155 154 F F T <---
99912024-01-24T19:09:43.149Z 22 440-459 92 92 89 155 155 148 F F T <---
99922024-01-24T19:09:43.149Z 23 460-479 75 75 75 122 122 122 F F F
99932024-01-24T19:09:43.150Z 24 480-499 85 85 85 142 142 142 T T F <---
99942024-01-24T19:09:43.150Z 25 500-519 88 88 88 146 146 146 F F F
99952024-01-24T19:09:43.150Z 26 520-539 92 92 89 159 159 148 T T F <---
99962024-01-24T19:09:43.150Z 27 540-559 92 92 83 157 157 137 F F F <---
99972024-01-24T19:09:43.150Z 28 560-579 92 92 84 155 155 140 F F F <---
99982024-01-24T19:09:43.150Z 29 580-599 90 90 90 149 149 149 F F F
99992024-01-24T19:09:43.150ZMax gen: 92, Max flush: 159
100002024-01-24T19:09:43.151ZError: Difference in extent metadata found!
100012024-01-24T19:09:43.151ZA Difference in extent metadata is expected here
100022024-01-24T19:09:43.151ZOn loop 46
100032024-01-24T19:09:43.151Z
100042024-01-24T19:09:43.151ZVerifying data now
100052024-01-24T19:09:49.392ZNo change after verify
100062024-01-24T19:09:49.392ZLoop: 46 Downstairs dump after verify (and repair):
100072024-01-24T19:09:49.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100082024-01-24T19:09:49.415Z 0 000-019 92 92 92 157 157 157 F F F
100092024-01-24T19:09:49.416Z 1 020-039 91 91 91 154 154 154 F F F
100102024-01-24T19:09:49.416Z 2 040-059 91 91 91 154 154 154 F F F
100112024-01-24T19:09:49.416Z 3 060-079 83 83 83 137 137 137 F F F
100122024-01-24T19:09:49.416Z 4 080-099 92 92 92 155 155 155 F F F
100132024-01-24T19:09:49.420Z 5 100-119 82 82 82 135 135 135 F F F
100142024-01-24T19:09:49.420Z 6 120-139 87 87 87 144 144 144 F F F
100152024-01-24T19:09:49.420Z 7 140-159 87 87 87 144 144 144 F F F
100162024-01-24T19:09:49.420Z 8 160-179 88 88 88 147 147 147 F F F
100172024-01-24T19:09:49.420Z 9 180-199 92 92 92 155 155 155 F F F
100182024-01-24T19:09:49.420Z 10 200-219 83 83 83 137 137 137 F F F
100192024-01-24T19:09:49.420Z 11 220-239 92 92 92 157 157 157 F F F
100202024-01-24T19:09:49.420Z 12 240-259 91 91 91 154 154 154 F F F
100212024-01-24T19:09:49.420Z 13 260-279 92 92 92 159 159 159 F F F
100222024-01-24T19:09:49.420Z 14 280-299 92 92 92 159 159 159 F F F
100232024-01-24T19:09:49.421Z 15 300-319 89 89 89 148 148 148 F F F
100242024-01-24T19:09:49.421Z 16 320-339 89 89 89 148 148 148 F F F
100252024-01-24T19:09:49.421Z 17 340-359 93 93 93 160 160 160 F F F
100262024-01-24T19:09:49.421Z 18 360-379 80 80 80 133 133 133 F F F
100272024-01-24T19:09:49.421Z 19 380-399 92 92 92 157 157 157 F F F
100282024-01-24T19:09:49.421Z 20 400-419 92 92 92 157 157 157 F F F
100292024-01-24T19:09:49.421Z 21 420-439 92 92 92 155 155 155 F F F
100302024-01-24T19:09:49.421Z 22 440-459 92 92 92 155 155 155 F F F
100312024-01-24T19:09:49.421Z 23 460-479 75 75 75 122 122 122 F F F
100322024-01-24T19:09:49.421Z 24 480-499 93 93 93 160 160 160 F F F
100332024-01-24T19:09:49.422Z 25 500-519 88 88 88 146 146 146 F F F
100342024-01-24T19:09:49.422Z 26 520-539 93 93 93 160 160 160 F F F
100352024-01-24T19:09:49.423Z 27 540-559 92 92 92 157 157 157 F F F
100362024-01-24T19:09:49.423Z 28 560-579 92 92 92 155 155 155 F F F
100372024-01-24T19:09:49.423Z 29 580-599 90 90 90 149 149 149 F F F
100382024-01-24T19:09:49.423ZMax gen: 93, Max flush: 160
100392024-01-24T19:09:49.426Z
100402024-01-24T19:09:49.426ZBegin loop 47
100412024-01-24T19:09:49.426ZDownstairs to restart: 1
100422024-01-24T19:09:49.472Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.118883819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505}
100432024-01-24T19:09:49.472Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.119404353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505}
100442024-01-24T19:09:49.472Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.119436938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505}
100452024-01-24T19:09:49.479Z{"msg":"Crucible f51d1d84-29ab-4335-9a21-20119cf1c915 has session id: c6ff031c-c1e6-4385-9e6a-40b9688588a2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.119497038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100462024-01-24T19:09:49.479Z{"msg":"Upstairs opts: Upstairs UUID: f51d1d84-29ab-4335-9a21-20119cf1c915, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
100472024-01-24T19:09:49.480Z,"time":"2024-01-24T19:09:48.119558571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100482024-01-24T19:09:49.480Z{"msg":"Crucible stats registered with UUID: f51d1d84-29ab-4335-9a21-20119cf1c915","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.119814212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100492024-01-24T19:09:49.480Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.119889852Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505}
100502024-01-24T19:09:49.480Z{"msg":"f51d1d84-29ab-4335-9a21-20119cf1c915 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120224879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100512024-01-24T19:09:49.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120296638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100522024-01-24T19:09:49.480Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.12034489Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100532024-01-24T19:09:49.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120439566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100542024-01-24T19:09:49.480Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120483015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100552024-01-24T19:09:49.481Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120528635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100562024-01-24T19:09:49.481Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120574885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100572024-01-24T19:09:49.482Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120749646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100582024-01-24T19:09:49.482Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.120986704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100592024-01-24T19:09:49.482Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.121072131Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100602024-01-24T19:09:49.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.121105711Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100612024-01-24T19:09:49.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.121129038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100622024-01-24T19:09:49.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.121215405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100632024-01-24T19:09:49.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.121899608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100642024-01-24T19:09:49.482Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.122056823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100652024-01-24T19:09:49.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.122181933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100662024-01-24T19:09:49.482Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.122280491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100672024-01-24T19:09:49.483Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.122831998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100682024-01-24T19:09:49.483Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.123029267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100692024-01-24T19:09:49.483Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.123112028Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100702024-01-24T19:09:49.483Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.123331259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100712024-01-24T19:09:49.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.123771433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100722024-01-24T19:09:49.483Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.12383913Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100732024-01-24T19:09:49.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.123867335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100742024-01-24T19:09:49.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.124311602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100752024-01-24T19:09:49.483Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.124370887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100762024-01-24T19:09:49.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.124402461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100772024-01-24T19:09:49.483Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.12443141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100782024-01-24T19:09:49.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125192612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100792024-01-24T19:09:49.483Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125256039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100802024-01-24T19:09:49.484Z{"msg":"[0]R flush_numbers[0..12]: [157, 154, 154, 137, 155, 135, 144, 144, 147, 155, 137, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125331258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100812024-01-24T19:09:49.484Z{"msg":"[0]R generation[0..12]: [92, 91, 91, 83, 92, 82, 87, 87, 88, 92, 83, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125408279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100822024-01-24T19:09:49.484Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125463756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100832024-01-24T19:09:49.484Z{"msg":"[1]R flush_numbers[0..12]: [157, 154, 154, 137, 155, 135, 144, 144, 147, 155, 137, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125490462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100842024-01-24T19:09:49.484Z{"msg":"[1]R generation[0..12]: [92, 91, 91, 83, 92, 82, 87, 87, 88, 92, 83, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125543342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100852024-01-24T19:09:49.484Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
100862024-01-24T19:09:49.484Ztime":"2024-01-24T19:09:48.125570101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100872024-01-24T19:09:49.486Z{"msg":"[2]R flush_numbers[0..12]: [157, 154, 154, 137, 155, 135, 144, 144, 147, 155, 137, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125693695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100882024-01-24T19:09:49.486Z{"msg":"[2]R generation[0..12]: [92, 91, 91, 83, 92, 82, 87, 87, 88, 92, 83, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125746644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100892024-01-24T19:09:49.486Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125774031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100902024-01-24T19:09:49.486Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125800809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100912024-01-24T19:09:49.486Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125827583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100922024-01-24T19:09:49.486Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125849464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100932024-01-24T19:09:49.486Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125871352Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100942024-01-24T19:09:49.486Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.12589348Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100952024-01-24T19:09:49.486Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125920244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100962024-01-24T19:09:49.487Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125970014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100972024-01-24T19:09:49.487Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.125999084Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100982024-01-24T19:09:49.487Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.126069426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
100992024-01-24T19:09:49.487Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.126094631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
101002024-01-24T19:09:49.487Z{"msg":"f51d1d84-29ab-4335-9a21-20119cf1c915 is now active with session: c6ff031c-c1e6-4385-9e6a-40b9688588a2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.126116514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
101012024-01-24T19:09:49.487Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.12616935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505,"session_id":"c6ff031c-c1e6-4385-9e6a-40b9688588a2"}
101022024-01-24T19:09:49.487Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:48.126228036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1505}
101032024-01-24T19:09:49.487ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101042024-01-24T19:09:49.487ZLoading write count information from file "/tmp/test_repair_verify.data"
101052024-01-24T19:09:49.487ZRun Repair workload
101062024-01-24T19:09:49.487Z01/30 Write block 537 len 32768 data: 8 8 8 6 8 10 10 11
101072024-01-24T19:09:49.487Z02/30 Write block 427 len 32768 data: 8 9 10 9 10 12 12 12
101082024-01-24T19:09:49.487Z03/30 Write block 568 len 4096 data: 9
101092024-01-24T19:09:49.487Z04/30 Flush
101102024-01-24T19:09:49.487Z05/30 Write block 514 len 24576 data: 9 10 8 10 11 11
101112024-01-24T19:09:49.487Z06/30 Read block 131 len 40960
101122024-01-24T19:09:49.487Z07/30 Write block 462 len 24576 data: 6 5 5 7 7 9
101132024-01-24T19:09:49.489Z08/30 Read block 270 len 16384
101142024-01-24T19:09:49.489Z09/30 Read block 343 len 20480
101152024-01-24T19:09:49.493Z10/30 Read block 40 len 28672
101162024-01-24T19:09:49.493Z11/30 Read block 270 len 16384
101172024-01-24T19:09:49.493Z12/30 Read block 115 len 4096
101182024-01-24T19:09:49.493Z13/30 Flush
101192024-01-24T19:09:49.493Z14/30 Write block 328 len 20480 data: 7 8 7 7 9
101202024-01-24T19:09:49.493Z15/30 Flush
101212024-01-24T19:09:49.493Z16/30 Flush
101222024-01-24T19:09:49.493Z17/30 Write block 124 len 20480 data: 10 9 8 7 6
101232024-01-24T19:09:49.493Z18/30 Read block 471 len 24576
101242024-01-24T19:09:49.493Z19/30 Read block 537 len 4096
101252024-01-24T19:09:49.493Z20/30 Read block 58 len 16384
101262024-01-24T19:09:49.497Z21/30 Read block 521 len 12288
101272024-01-24T19:09:49.497Z22/30 Read block 124 len 24576
101282024-01-24T19:09:49.500Z23/30 Write block 123 len 8192 data: 7 11
101292024-01-24T19:09:49.500Z24/30 Read block 468 len 28672
101302024-01-24T19:09:49.500Z25/30 Read block 375 len 40960
101312024-01-24T19:09:49.503Z26/30 Read block 227 len 20480
101322024-01-24T19:09:49.507Z27/30 Write block 380 len 28672 data: 8 7 5 6 5 4 6
101332024-01-24T19:09:49.507Z28/30 Write block 77 len 20480 data: 8 9 10 10 12
101342024-01-24T19:09:49.507Z29/30 Write block 209 len 24576 data: 7 9 7 7 6 5
101352024-01-24T19:09:49.511Z30/30 Read block 248 len 4096
101362024-01-24T19:09:49.511Z----------------------------------------------------------------
101372024-01-24T19:09:49.511Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
101382024-01-24T19:09:49.511ZStates: Active Active Active
101392024-01-24T19:09:49.511ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101402024-01-24T19:09:49.513Z 1 Acked 1000 Write 8 Done Sent Done false
101412024-01-24T19:09:49.513Z 2 Acked 1001 Write 8 Done Sent Done false
101422024-01-24T19:09:49.513Z 3 Acked 1002 Write 1 Done Sent Done false
101432024-01-24T19:09:49.513Z 4 Acked 1003 Flush 0 Done Sent Done false
101442024-01-24T19:09:49.513Z 5 Acked 1004 Write 6 Done Sent Done false
101452024-01-24T19:09:49.513Z 6 Acked 1005 Read 10 Done Sent Done false
101462024-01-24T19:09:49.513Z 7 Acked 1006 Write 6 Done Sent Done false
101472024-01-24T19:09:49.513Z 8 Acked 1007 Read 4 Done Sent Done false
101482024-01-24T19:09:49.513Z 9 Acked 1008 Read 5 Done Sent Done false
101492024-01-24T19:09:49.513Z 10 Acked 1009 Read 7 Done Sent Done false
101502024-01-24T19:09:49.513Z 11 Acked 1010 Read 4 Done Sent Done false
101512024-01-24T19:09:49.513Z 12 Acked 1011 Read 1 Done Sent Done false
101522024-01-24T19:09:49.513Z 13 Acked 1012 Flush 0 Done Sent Done false
101532024-01-24T19:09:49.513Z 14 Acked 1013 Write 5 Done Sent Done false
101542024-01-24T19:09:49.513Z 15 Acked 1014 Flush 0 Done Sent Done false
101552024-01-24T19:09:49.513Z 16 Acked 1015 Flush 0 Done Sent Done false
101562024-01-24T19:09:49.513Z 17 Acked 1016 Write 5 Done Sent Done false
101572024-01-24T19:09:49.514Z 18 Acked 1017 Read 6 Done Sent Done false
101582024-01-24T19:09:49.514Z 19 Acked 1018 Read 1 Done Sent Done false
101592024-01-24T19:09:49.514Z 20 Acked 1019 Read 4 Done Sent Done false
101602024-01-24T19:09:49.514Z 21 Acked 1020 Read 3 Done Sent Done false
101612024-01-24T19:09:49.514Z 22 Acked 1021 Read 6 Done Sent Done false
101622024-01-24T19:09:49.514Z 23 Acked 1022 Write 2 Done Sent Done false
101632024-01-24T19:09:49.514Z 24 Acked 1023 Read 7 Done Sent Done false
101642024-01-24T19:09:49.514Z 25 Acked 1024 Read 10 Done Sent Done false
101652024-01-24T19:09:49.514Z 26 Acked 1025 Read 5 Done Sent Done false
101662024-01-24T19:09:49.514Z 27 Acked 1026 Write 7 Done Sent Done false
101672024-01-24T19:09:49.514Z 28 Acked 1027 Write 5 Done Sent Done false
101682024-01-24T19:09:49.514Z 29 Acked 1028 Write 6 Done Sent Done false
101692024-01-24T19:09:49.514Z 30 Acked 1029 Read 1 Done Sent Done false
101702024-01-24T19:09:49.514Z STATES DS:0 DS:1 DS:2 TOTAL
101712024-01-24T19:09:49.514Z New 0 0 0 0
101722024-01-24T19:09:49.514Z Sent 0 30 0 30
101732024-01-24T19:09:49.514Z Done 30 0 30 60
101742024-01-24T19:09:49.514Z Skipped 0 0 0 0
101752024-01-24T19:09:49.514Z Error 0 0 0 0
101762024-01-24T19:09:49.514ZLast Flush: 1015 0 1015
101772024-01-24T19:09:49.514ZDownstairs last five completed:
101782024-01-24T19:09:49.515ZUpstairs last five completed: 30 29 28 27 26
101792024-01-24T19:09:49.515ZWrote out file "/tmp/test_repair_verify.data"
101802024-01-24T19:09:49.519Z
101812024-01-24T19:09:49.523ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101822024-01-24T19:09:49.611ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101832024-01-24T19:09:49.611Z 0 000-019 92 92 92 157 157 157 F F F
101842024-01-24T19:09:49.616Z 1 020-039 91 91 91 154 154 154 F F F
101852024-01-24T19:09:49.616Z 2 040-059 91 91 91 154 154 154 F F F
101862024-01-24T19:09:49.616Z 3 060-079 83 83 83 137 137 137 T F T <---
101872024-01-24T19:09:49.616Z 4 080-099 92 92 92 155 155 155 T F T <---
101882024-01-24T19:09:49.616Z 5 100-119 82 82 82 135 135 135 F F F
101892024-01-24T19:09:49.616Z 6 120-139 87 87 87 144 144 144 T F T <---
101902024-01-24T19:09:49.616Z 7 140-159 87 87 87 144 144 144 F F F
101912024-01-24T19:09:49.617Z 8 160-179 88 88 88 147 147 147 F F F
101922024-01-24T19:09:49.617Z 9 180-199 92 92 92 155 155 155 F F F
101932024-01-24T19:09:49.617Z 10 200-219 83 83 83 137 137 137 T F T <---
101942024-01-24T19:09:49.617Z 11 220-239 92 92 92 157 157 157 F F F
101952024-01-24T19:09:49.617Z 12 240-259 91 91 91 154 154 154 F F F
101962024-01-24T19:09:49.617Z 13 260-279 92 92 92 159 159 159 F F F
101972024-01-24T19:09:49.617Z 14 280-299 92 92 92 159 159 159 F F F
101982024-01-24T19:09:49.617Z 15 300-319 89 89 89 148 148 148 F F F
101992024-01-24T19:09:49.617Z 16 320-339 94 89 94 163 148 163 F F F <---
102002024-01-24T19:09:49.617Z 17 340-359 93 93 93 160 160 160 F F F
102012024-01-24T19:09:49.617Z 18 360-379 80 80 80 133 133 133 F F F
102022024-01-24T19:09:49.617Z 19 380-399 92 92 92 157 157 157 T F T <---
102032024-01-24T19:09:49.617Z 20 400-419 92 92 92 157 157 157 F F F
102042024-01-24T19:09:49.617Z 21 420-439 94 92 94 161 155 161 F F F <---
102052024-01-24T19:09:49.617Z 22 440-459 92 92 92 155 155 155 F F F
102062024-01-24T19:09:49.617Z 23 460-479 94 75 94 162 122 162 F F F <---
102072024-01-24T19:09:49.620Z 24 480-499 93 93 93 160 160 160 F F F
102082024-01-24T19:09:49.620Z 25 500-519 94 88 94 162 146 162 F F F <---
102092024-01-24T19:09:49.620Z 26 520-539 94 93 94 161 160 161 F F F <---
102102024-01-24T19:09:49.621Z 27 540-559 94 92 94 161 157 161 F F F <---
102112024-01-24T19:09:49.621Z 28 560-579 94 92 94 161 155 161 F F F <---
102122024-01-24T19:09:49.621Z 29 580-599 90 90 90 149 149 149 F F F
102132024-01-24T19:09:49.621ZMax gen: 94, Max flush: 163
102142024-01-24T19:09:49.621ZError: Difference in extent metadata found!
102152024-01-24T19:09:49.621ZA Difference in extent metadata is expected here
102162024-01-24T19:09:49.621ZOn loop 47
102172024-01-24T19:09:49.621Z
102182024-01-24T19:09:49.621ZVerifying data now
102192024-01-24T19:09:54.625ZNo change after verify
102202024-01-24T19:09:54.626ZLoop: 47 Downstairs dump after verify (and repair):
102212024-01-24T19:09:54.660ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102222024-01-24T19:09:54.660Z 0 000-019 92 92 92 157 157 157 F F F
102232024-01-24T19:09:54.688Z 1 020-039 91 91 91 154 154 154 F F F
102242024-01-24T19:09:54.688Z 2 040-059 91 91 91 154 154 154 F F F
102252024-01-24T19:09:54.688Z 3 060-079 95 95 95 164 164 164 F F F
102262024-01-24T19:09:54.688Z 4 080-099 95 95 95 164 164 164 F F F
102272024-01-24T19:09:54.688Z 5 100-119 82 82 82 135 135 135 F F F
102282024-01-24T19:09:54.688Z 6 120-139 95 95 95 164 164 164 F F F
102292024-01-24T19:09:54.688Z 7 140-159 87 87 87 144 144 144 F F F
102302024-01-24T19:09:54.688Z 8 160-179 88 88 88 147 147 147 F F F
102312024-01-24T19:09:54.688Z 9 180-199 92 92 92 155 155 155 F F F
102322024-01-24T19:09:54.688Z 10 200-219 95 95 95 164 164 164 F F F
102332024-01-24T19:09:54.689Z 11 220-239 92 92 92 157 157 157 F F F
102342024-01-24T19:09:54.689Z 12 240-259 91 91 91 154 154 154 F F F
102352024-01-24T19:09:54.689Z 13 260-279 92 92 92 159 159 159 F F F
102362024-01-24T19:09:54.689Z 14 280-299 92 92 92 159 159 159 F F F
102372024-01-24T19:09:54.689Z 15 300-319 89 89 89 148 148 148 F F F
102382024-01-24T19:09:54.689Z 16 320-339 94 94 94 163 163 163 F F F
102392024-01-24T19:09:54.689Z 17 340-359 93 93 93 160 160 160 F F F
102402024-01-24T19:09:54.690Z 18 360-379 80 80 80 133 133 133 F F F
102412024-01-24T19:09:54.690Z 19 380-399 95 95 95 164 164 164 F F F
102422024-01-24T19:09:54.690Z 20 400-419 92 92 92 157 157 157 F F F
102432024-01-24T19:09:54.690Z 21 420-439 94 94 94 161 161 161 F F F
102442024-01-24T19:09:54.690Z 22 440-459 92 92 92 155 155 155 F F F
102452024-01-24T19:09:54.690Z 23 460-479 94 94 94 162 162 162 F F F
102462024-01-24T19:09:54.691Z 24 480-499 93 93 93 160 160 160 F F F
102472024-01-24T19:09:54.691Z 25 500-519 94 94 94 162 162 162 F F F
102482024-01-24T19:09:54.691Z 26 520-539 94 94 94 161 161 161 F F F
102492024-01-24T19:09:54.692Z 27 540-559 94 94 94 161 161 161 F F F
102502024-01-24T19:09:54.692Z 28 560-579 94 94 94 161 161 161 F F F
102512024-01-24T19:09:54.692Z 29 580-599 90 90 90 149 149 149 F F F
102522024-01-24T19:09:54.692ZMax gen: 95, Max flush: 164
102532024-01-24T19:09:54.692Z
102542024-01-24T19:09:54.692ZBegin loop 48
102552024-01-24T19:09:54.693ZDownstairs to restart: 0
102562024-01-24T19:09:54.726Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.346916521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513}
102572024-01-24T19:09:54.726Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347393573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513}
102582024-01-24T19:09:54.726Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347439751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513}
102592024-01-24T19:09:54.727Z{"msg":"Crucible 8fce7a60-5d58-4515-a6d6-f705b84929d1 has session id: 73059e74-8ef0-40b7-91cb-eba79018a853","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347507825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513Crucible runtime is spawned
102602024-01-24T19:09:54.727Z,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102612024-01-24T19:09:54.727Z{"msg":"Upstairs opts: Upstairs UUID: 8fce7a60-5d58-4515-a6d6-f705b84929d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347596837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102622024-01-24T19:09:54.727Z{"msg":"Crucible stats registered with UUID: 8fce7a60-5d58-4515-a6d6-f705b84929d1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347656966Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102632024-01-24T19:09:54.728Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347683397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513}
102642024-01-24T19:09:54.728Z{"msg":"8fce7a60-5d58-4515-a6d6-f705b84929d1 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347869362Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102652024-01-24T19:09:54.728Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347922263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102662024-01-24T19:09:54.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347946356Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102672024-01-24T19:09:54.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347968396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102682024-01-24T19:09:54.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.347990656Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102692024-01-24T19:09:54.729Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.34802649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102702024-01-24T19:09:54.729Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348065607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102712024-01-24T19:09:54.730Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348185114Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102722024-01-24T19:09:54.730Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348384797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102732024-01-24T19:09:54.730Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348593659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102742024-01-24T19:09:54.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348642312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102752024-01-24T19:09:54.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348718436Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102762024-01-24T19:09:54.731Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.348784398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102772024-01-24T19:09:54.731Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.349934396Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102782024-01-24T19:09:54.731Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.350081335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102792024-01-24T19:09:54.732Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.350155811Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102802024-01-24T19:09:54.732Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.350187016Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102812024-01-24T19:09:54.732Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.350333226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102822024-01-24T19:09:54.732Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.350404308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102832024-01-24T19:09:54.734Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.351125565Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102842024-01-24T19:09:54.734Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.351195183Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102852024-01-24T19:09:54.734Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.351238968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102862024-01-24T19:09:54.734Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.351986756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102872024-01-24T19:09:54.735Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352087039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102882024-01-24T19:09:54.735Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352136415Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102892024-01-24T19:09:54.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352252306Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102902024-01-24T19:09:54.735Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352301078Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102912024-01-24T19:09:54.735Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352345028Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102922024-01-24T19:09:54.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352418108Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102932024-01-24T19:09:54.736Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352492048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102942024-01-24T19:09:54.736Z{"msg":"[0]R flush_numbers[0..12]: [157, 154, 154, 164, 164, 135, 164, 144, 147, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352568485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102952024-01-24T19:09:54.737Z{"msg":"[0]R generation[0..12]: [92, 91, 91, 95, 95, 82, 95, 87, 88, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352643624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102962024-01-24T19:09:54.737Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352719031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102972024-01-24T19:09:54.737Z{"msg"Wait for a query_work_queue command to finish before sending IO
102982024-01-24T19:09:54.737Z:"[1]R flush_numbers[0..12]: [157, 154, 154, 164, 164, 135, 164, 144, 147, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.3527953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
102992024-01-24T19:09:54.737Z{"msg":"[1]R generation[0..12]: [92, 91, 91, 95, 95, 82, 95, 87, 88, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352871231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103002024-01-24T19:09:54.737Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.352950918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103012024-01-24T19:09:54.737Z{"msg":"[2]R flush_numbers[0..12]: [157, 154, 154, 164, 164, 135, 164, 144, 147, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353031734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103022024-01-24T19:09:54.737Z{"msg":"[2]R generation[0..12]: [92, 91, 91, 95, 95, 82, 95, 87, 88, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353109874Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103032024-01-24T19:09:54.739Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.35318856Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103042024-01-24T19:09:54.739Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353261341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103052024-01-24T19:09:54.739Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353335319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103062024-01-24T19:09:54.739Z{"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353414142Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103072024-01-24T19:09:54.739Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103082024-01-24T19:09:54.739Z,"time":"2024-01-24T19:09:53.35349283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103092024-01-24T19:09:54.740Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353572606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103102024-01-24T19:09:54.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353625145Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103112024-01-24T19:09:54.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353650586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103122024-01-24T19:09:54.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353693593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103132024-01-24T19:09:54.740Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353718935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103142024-01-24T19:09:54.740Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353759818Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103152024-01-24T19:09:54.740Z{"msg":"8fce7a60-5d58-4515-a6d6-f705b84929d1 is now active with session: 73059e74-8ef0-40b7-91cb-eba79018a853","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353784662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103162024-01-24T19:09:54.740Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353824895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513,"session_id":"73059e74-8ef0-40b7-91cb-eba79018a853"}
103172024-01-24T19:09:54.740Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:53.353848525Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1513}
103182024-01-24T19:09:54.740ZLoading write count information from file "/tmp/test_repair_verify.data"
103192024-01-24T19:09:54.740ZRun Repair workload
103202024-01-24T19:09:54.740Z01/30 Write block 118 len 12288 data: 5 7 7
103212024-01-24T19:09:54.741Z02/30 Flush
103222024-01-24T19:09:54.741Z03/30 Write block 245 len 24576 data: 10 12 11 10 11 8
103232024-01-24T19:09:54.741Z04/30 Read block 333 len 16384
103242024-01-24T19:09:54.741Z05/30 Read block 514 len 8192
103252024-01-24T19:09:54.741Z06/30 Read block 551 len 32768
103262024-01-24T19:09:54.741Z07/30 Read block 185 len 20480
103272024-01-24T19:09:55.963Z08/30 Write block 29 len 12288 data: 11 11 13
103282024-01-24T19:09:55.963Z09/30 Flush
103292024-01-24T19:09:55.963Z10/30 Write block 509 len 36864 data: 8 12 12 12 11 10 11 9 11
103302024-01-24T19:09:55.963Z11/30 Write block 102 len 20480 data: 4 8 9 9 9
103312024-01-24T19:09:55.964Z12/30 Write block 355 len 12288 data: 5 7 6
103322024-01-24T19:09:55.964Z13/30 Write block 61 len 16384 data: 6 5 6 8
103332024-01-24T19:09:55.964Z14/30 Read block 411 len 8192
103342024-01-24T19:09:55.964Z15/30 Write block 165 len 24576 data: 10 8 9 11 9 8
103352024-01-24T19:09:55.964Z16/30 Read block 480 len 32768
103362024-01-24T19:09:55.965Z17/30 Read block 44 len 28672
103372024-01-24T19:09:55.965Z18/30 Flush
103382024-01-24T19:09:55.965Z19/30 Flush
103392024-01-24T19:09:55.965Z20/30 Write block 375 len 36864 data: 10 9 9 6 6 9 8 6 7
103402024-01-24T19:09:55.965Z21/30 Write block 408 len 20480 data: 7 4 4 4 4
103412024-01-24T19:09:55.965Z22/30 Write block 132 len 4096 data: 9
103422024-01-24T19:09:55.965Z23/30 Write block 168 len 12288 data: 12 10 9
103432024-01-24T19:09:55.965Z24/30 Read block 550 len 24576
103442024-01-24T19:09:55.965Z25/30 Read block 342 len 40960
103452024-01-24T19:09:55.965Z26/30 Write block 250 len 8192 data: 9 7
103462024-01-24T19:09:55.965Z27/30 Read block 207 len 4096
103472024-01-24T19:09:55.965Z28/30 Read block 250 len 24576
103482024-01-24T19:09:55.965Z29/30 Read block 3 len 36864
103492024-01-24T19:09:55.965Z30/30 Write block 404 len 20480 data: 4 7 7 7 8
103502024-01-24T19:09:55.966Z----------------------------------------------------------------
103512024-01-24T19:09:55.966Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:28
103522024-01-24T19:09:55.966ZStates: Active Active Active
103532024-01-24T19:09:55.966ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103542024-01-24T19:09:55.966Z 3 Acked 1002 Write 6 Done Done Done false
103552024-01-24T19:09:55.966Z 4 Acked 1003 Read 4 Done Done Done false
103562024-01-24T19:09:55.966Z 5 Acked 1004 Read 2 Done Done Done false
103572024-01-24T19:09:55.966Z 6 Acked 1005 Read 8 Sent Done Done false
103582024-01-24T19:09:55.966Z 7 Acked 1006 Read 5 Sent Done Done false
103592024-01-24T19:09:55.966Z 8 Acked 1007 Write 3 Sent Done Done false
103602024-01-24T19:09:55.966Z 9 Acked 1008 Flush 0 Sent Done Done false
103612024-01-24T19:09:55.967Z 10 Acked 1009 Write 9 Sent Done Done false
103622024-01-24T19:09:55.967Z 11 Acked 1010 Write 5 Sent Done Done false
103632024-01-24T19:09:55.967Z 12 Acked 1011 Write 3 Sent Done Done false
103642024-01-24T19:09:55.967Z 13 Acked 1012 Write 4 Sent Done Done false
103652024-01-24T19:09:55.967Z 14 Acked 1013 Read 2 Sent Done Done false
103662024-01-24T19:09:55.967Z 15 Acked 1014 Write 6 Sent Done Done false
103672024-01-24T19:09:55.967Z 16 Acked 1015 Read 8 Sent Done Done false
103682024-01-24T19:09:55.967Z 17 Acked 1016 Read 7 Sent Done Done false
103692024-01-24T19:09:55.967Z 18 Acked 1017 Flush 0 Sent Done Done false
103702024-01-24T19:09:55.967Z 19 Acked 1018 Flush 0 Sent Done Done false
103712024-01-24T19:09:55.967Z 20 Acked 1019 Write 9 Sent Done Done false
103722024-01-24T19:09:55.967Z 21 Acked 1020 Write 5 Sent Done Done false
103732024-01-24T19:09:55.967Z 22 Acked 1021 Write 1 Sent Done Done false
103742024-01-24T19:09:55.967Z 23 Acked 1022 Write 3 Sent Done Done false
103752024-01-24T19:09:55.967Z 24 Acked 1023 Read 6 Sent Done Done false
103762024-01-24T19:09:55.967Z 25 Acked 1024 Read 10 Sent Done Done false
103772024-01-24T19:09:55.968Z 26 Acked 1025 Write 2 Sent Done Done false
103782024-01-24T19:09:55.968Z 27 Acked 1026 Read 1 Sent Done Done false
103792024-01-24T19:09:55.968Z 28 Acked 1027 Read 6 Sent Done Done false
103802024-01-24T19:09:55.968Z 29 Acked 1028 Read 9 Sent Done Done false
103812024-01-24T19:09:55.968Z 30 Acked 1029 Write 5 Sent Sent Sent false
103822024-01-24T19:09:55.968Z STATES DS:0 DS:1 DS:2 TOTAL
103832024-01-24T19:09:55.968Z New 0 0 0 0
103842024-01-24T19:09:55.969Z Sent 25 1 1 27
103852024-01-24T19:09:55.969Z Done 3 27 27 57
103862024-01-24T19:09:55.969Z Skipped 0 0 0 0
103872024-01-24T19:09:55.969Z Error 0 0 0 0
103882024-01-24T19:09:55.969ZLast Flush: 1001 1018 1018
103892024-01-24T19:09:55.969ZDownstairs last five completed: 1001 1000
103902024-01-24T19:09:55.969ZUpstairs last five completed: 30 29 28 27 26
103912024-01-24T19:09:55.969ZWrote out file "/tmp/test_repair_verify.data"
103922024-01-24T19:09:55.969Z
103932024-01-24T19:09:55.969ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
103942024-01-24T19:09:55.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103952024-01-24T19:09:55.979Z 0 000-019 92 92 92 157 157 157 F F F
103962024-01-24T19:09:55.979Z 1 020-039 91 96 96 154 166 166 F F F <---
103972024-01-24T19:09:55.979Z 2 040-059 91 91 91 154 154 154 F F F
103982024-01-24T19:09:55.979Z 3 060-079 95 96 96 164 167 167 F F F <---
103992024-01-24T19:09:55.979Z 4 080-099 95 95 95 164 164 164 F F F
104002024-01-24T19:09:55.980Z 5 100-119 96 96 96 165 167 167 F F F <---
104012024-01-24T19:09:55.980Z 6 120-139 96 96 96 165 165 165 F T T <---
104022024-01-24T19:09:55.980Z 7 140-159 87 87 87 144 144 144 F F F
104032024-01-24T19:09:55.980Z 8 160-179 88 96 96 147 167 167 F T T <---
104042024-01-24T19:09:55.980Z 9 180-199 92 92 92 155 155 155 F F F
104052024-01-24T19:09:55.980Z 10 200-219 95 95 95 164 164 164 F F F
104062024-01-24T19:09:55.980Z 11 220-239 92 92 92 157 157 157 F F F
104072024-01-24T19:09:55.980Z 12 240-259 91 96 96 154 166 166 T T T <---
104082024-01-24T19:09:55.980Z 13 260-279 92 92 92 159 159 159 F F F
104092024-01-24T19:09:55.980Z 14 280-299 92 92 92 159 159 159 F F F
104102024-01-24T19:09:55.980Z 15 300-319 89 89 89 148 148 148 F F F
104112024-01-24T19:09:55.980Z 16 320-339 94 94 94 163 163 163 F F F
104122024-01-24T19:09:55.980Z 17 340-359 93 96 96 160 167 167 F F F <---
104132024-01-24T19:09:55.980Z 18 360-379 80 80 80 133 133 133 F T T <---
104142024-01-24T19:09:55.980Z 19 380-399 95 95 95 164 164 164 F T T <---
104152024-01-24T19:09:55.980Z 20 400-419 92 92 92 157 157 157 F T T <---
104162024-01-24T19:09:55.980Z 21 420-439 94 94 94 161 161 161 F F F
104172024-01-24T19:09:55.980Z 22 440-459 92 92 92 155 155 155 F F F
104182024-01-24T19:09:55.980Z 23 460-479 94 94 94 162 162 162 F F F
104192024-01-24T19:09:55.980Z 24 480-499 93 93 93 160 160 160 F F F
104202024-01-24T19:09:55.980Z 25 500-519 94 96 96 162 167 167 F F F <---
104212024-01-24T19:09:55.980Z 26 520-539 94 94 94 161 161 161 F F F
104222024-01-24T19:09:55.981Z 27 540-559 94 94 94 161 161 161 F F F
104232024-01-24T19:09:55.983Z 28 560-579 94 94 94 161 161 161 F F F
104242024-01-24T19:09:55.983Z 29 580-599 90 90 90 149 149 149 F F F
104252024-01-24T19:09:55.983ZMax gen: 96, Max flush: 167
104262024-01-24T19:09:55.983ZError: Difference in extent metadata found!
104272024-01-24T19:09:55.983ZA Difference in extent metadata is expected here
104282024-01-24T19:09:55.983ZOn loop 48
104292024-01-24T19:09:55.983Z
104302024-01-24T19:09:55.983ZVerifying data now
104312024-01-24T19:09:59.794ZNo change after verify
104322024-01-24T19:09:59.794ZLoop: 48 Downstairs dump after verify (and repair):
104332024-01-24T19:09:59.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104342024-01-24T19:09:59.817Z 0 000-019 92 92 92 157 157 157 F F F
104352024-01-24T19:09:59.821Z 1 020-039 96 96 96 166 166 166 F F F
104362024-01-24T19:09:59.821Z 2 040-059 91 91 91 154 154 154 F F F
104372024-01-24T19:09:59.821Z 3 060-079 96 96 96 167 167 167 F F F
104382024-01-24T19:09:59.821Z 4 080-099 95 95 95 164 164 164 F F F
104392024-01-24T19:09:59.821Z 5 100-119 96 96 96 167 167 167 F F F
104402024-01-24T19:09:59.821Z 6 120-139 97 97 97 168 168 168 F F F
104412024-01-24T19:09:59.821Z 7 140-159 87 87 87 144 144 144 F F F
104422024-01-24T19:09:59.821Z 8 160-179 97 97 97 168 168 168 F F F
104432024-01-24T19:09:59.821Z 9 180-199 92 92 92 155 155 155 F F F
104442024-01-24T19:09:59.821Z 10 200-219 95 95 95 164 164 164 F F F
104452024-01-24T19:09:59.821Z 11 220-239 92 92 92 157 157 157 F F F
104462024-01-24T19:09:59.821Z 12 240-259 97 97 97 168 168 168 F F F
104472024-01-24T19:09:59.821Z 13 260-279 92 92 92 159 159 159 F F F
104482024-01-24T19:09:59.821Z 14 280-299 92 92 92 159 159 159 F F F
104492024-01-24T19:09:59.821Z 15 300-319 89 89 89 148 148 148 F F F
104502024-01-24T19:09:59.821Z 16 320-339 94 94 94 163 163 163 F F F
104512024-01-24T19:09:59.821Z 17 340-359 96 96 96 167 167 167 F F F
104522024-01-24T19:09:59.821Z 18 360-379 97 97 97 168 168 168 F F F
104532024-01-24T19:09:59.822Z 19 380-399 97 97 97 168 168 168 F F F
104542024-01-24T19:09:59.822Z 20 400-419 97 97 97 168 168 168 F F F
104552024-01-24T19:09:59.822Z 21 420-439 94 94 94 161 161 161 F F F
104562024-01-24T19:09:59.822Z 22 440-459 92 92 92 155 155 155 F F F
104572024-01-24T19:09:59.822Z 23 460-479 94 94 94 162 162 162 F F F
104582024-01-24T19:09:59.822Z 24 480-499 93 93 93 160 160 160 F F F
104592024-01-24T19:09:59.822Z 25 500-519 96 96 96 167 167 167 F F F
104602024-01-24T19:09:59.822Z 26 520-539 94 94 94 161 161 161 F F F
104612024-01-24T19:09:59.822Z 27 540-559 94 94 94 161 161 161 F F F
104622024-01-24T19:09:59.822Z 28 560-579 94 94 94 161 161 161 F F F
104632024-01-24T19:09:59.822Z 29 580-599 90 90 90 149 149 149 F F F
104642024-01-24T19:09:59.822ZMax gen: 97, Max flush: 168
104652024-01-24T19:09:59.826Z
104662024-01-24T19:09:59.826ZBegin loop 49
104672024-01-24T19:09:59.826ZDownstairs to restart: 2
104682024-01-24T19:09:59.854Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521137788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521}
104692024-01-24T19:09:59.854Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521714796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521}
104702024-01-24T19:09:59.856Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521745877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521}
104712024-01-24T19:09:59.879Z{"msg":"Crucible b4c6dbe9-13d5-43d7-ac7c-b411be2c5a12 has session id: e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521812179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104722024-01-24T19:09:59.879Z{"msg":"Upstairs opts: Upstairs UUID: b4c6dbe9-13d5-43d7-ac7c-b411be2c5a12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521902102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104732024-01-24T19:09:59.879Z{"msg":"Crucible stats registered with UUID: b4c6dbe9-13d5-43d7-ac7c-b411be2c5a12","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.521976939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104742024-01-24T19:09:59.879ZCrucible runtime is spawned
104752024-01-24T19:09:59.880Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522270335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521}
104762024-01-24T19:09:59.880Z{"msg":"b4c6dbe9-13d5-43d7-ac7c-b411be2c5a12 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522430962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104772024-01-24T19:09:59.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522501218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104782024-01-24T19:09:59.880Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522547123Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104792024-01-24T19:09:59.880Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522639495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104802024-01-24T19:09:59.881Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522685688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104812024-01-24T19:09:59.881Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.52276564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104822024-01-24T19:09:59.881Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522809851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104832024-01-24T19:09:59.881Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522905824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104842024-01-24T19:09:59.881Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.522957086Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104852024-01-24T19:09:59.881Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.523000386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104862024-01-24T19:09:59.881Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.523042038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104872024-01-24T19:09:59.881Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.523133029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104882024-01-24T19:09:59.881Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.523174894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104892024-01-24T19:09:59.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.524748332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104902024-01-24T19:09:59.882Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.524826494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104912024-01-24T19:09:59.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.524855218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104922024-01-24T19:09:59.882Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.524881932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104932024-01-24T19:09:59.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.525243043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104942024-01-24T19:09:59.883Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.525318684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104952024-01-24T19:09:59.883Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.525939535Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104962024-01-24T19:09:59.883Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.526320613Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104972024-01-24T19:09:59.883Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.526579521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104982024-01-24T19:09:59.883Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.526879644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
104992024-01-24T19:09:59.883Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.526955798Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105002024-01-24T19:09:59.884Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527049503Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105012024-01-24T19:09:59.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527101418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105022024-01-24T19:09:59.891Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527127626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105032024-01-24T19:09:59.891Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527210649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105042024-01-24T19:09:59.891Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527456084Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105052024-01-24T19:09:59.891Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527555415Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105062024-01-24T19:09:59.891Z{"msg":"[0]R flush_numbers[0..12]: [157, 166, 154, 167, 164, 167, 168, 144, 168, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527642226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105072024-01-24T19:09:59.891Z{"msg":"[0]R generation[0..12]: [92, 96, 91, 96, 95, 96, 97, 87, 97, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.52767085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105082024-01-24T19:09:59.891Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527695294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105092024-01-24T19:09:59.891Z{"msg":"[1]R flush_numbers[0..12]: [157, 166, 154, 167, 164, 167, 168, 144, 168, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.52772757Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105102024-01-24T19:09:59.893Z{"msg":"[1]R generation[0..12]: [92, 96, 91, 96, 95, 96, 97, 87, 97, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527752307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105112024-01-24T19:09:59.893ZWait for a query_work_queue command to finish before sending IO
105122024-01-24T19:09:59.893Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527801201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105132024-01-24T19:09:59.893Z{"msg":"[2]R flush_numbers[0..12]: [157, 166, 154, 167, 164, 167, 168, 144, 168, 155, 164, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527828592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105142024-01-24T19:09:59.894Z{"msg":"[2]R generation[0..12]: [92, 96, 91, 96, 95, 96, 97, 87, 97, 92, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527852948Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105152024-01-24T19:09:59.894Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527880062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105162024-01-24T19:09:59.894Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527927997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105172024-01-24T19:09:59.894Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.527955332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105182024-01-24T19:09:59.894Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528002956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105192024-01-24T19:09:59.894Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528028303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105202024-01-24T19:09:59.894Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528051914Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105212024-01-24T19:09:59.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528072052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105222024-01-24T19:09:59.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528093866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105232024-01-24T19:09:59.894Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528143663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105242024-01-24T19:09:59.894Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528192502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105252024-01-24T19:09:59.894Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528216137Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105262024-01-24T19:09:59.895Z{"msg":"b4c6dbe9-13d5-43d7-ac7c-b411be2c5a12 is now active with session: e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528259418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105272024-01-24T19:09:59.895Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528282801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521,"session_id":"e6bb2d61-cc81-4e7b-8d7d-1a7a7996ced3"}
105282024-01-24T19:09:59.895Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:09:58.528303352Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1521}
105292024-01-24T19:09:59.895ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105302024-01-24T19:09:59.895ZLoading write count information from file "/tmp/test_repair_verify.data"
105312024-01-24T19:09:59.895ZRun Repair workload
105322024-01-24T19:09:59.896Z01/30 Write block 502 len 24576 data: 6 3 5 5 5 8
105332024-01-24T19:09:59.896Z02/30 Write block 361 len 32768 data: 10 9 8 10 10 10 9 9
105342024-01-24T19:09:59.896Z03/30 Write block 187 len 40960 data: 8 7 6 6 6 7 7 7 6 6
105352024-01-24T19:09:59.896Z04/30 Write block 497 len 40960 data: 10 10 9 6 8 7 4 6 6 6
105362024-01-24T19:09:59.896Z05/30 Write block 294 len 24576 data: 11 10 9 7 7 5
105372024-01-24T19:09:59.896Z06/30 Write block 274 len 40960 data: 10 9 7 10 11 13 11 11 11 11
105382024-01-24T19:09:59.896Z07/30 Read block 402 len 4096
105392024-01-24T19:09:59.896Z08/30 Write block 513 len 24576 data: 12 11 12 10 12 12
105402024-01-24T19:09:59.896Z09/30 Flush
105412024-01-24T19:09:59.897Z10/30 Write block 565 len 24576 data: 7 7 9 10 9 6
105422024-01-24T19:09:59.897Z11/30 Read block 482 len 32768
105432024-01-24T19:09:59.897Z12/30 Read block 208 len 40960
105442024-01-24T19:09:59.897Z13/30 Flush
105452024-01-24T19:09:59.902Z14/30 Write block 193 len 24576 data: 8 8 7 7 6 5
105462024-01-24T19:09:59.902Z15/30 Write block 277 len 36864 data: 11 12 14 12 12 12 12 8 6
105472024-01-24T19:09:59.902Z16/30 Read block 31 len 40960
105482024-01-24T19:09:59.902Z17/30 Read block 395 len 12288
105492024-01-24T19:09:59.902Z18/30 Read block 58 len 8192
105502024-01-24T19:09:59.902Z19/30 Write block 0 len 28672 data: 2 4 5 6 6 6 8
105512024-01-24T19:09:59.902Z20/30 Write block 7 len 16384 data: 9 11 8 8
105522024-01-24T19:09:59.902Z21/30 Read block 66 len 16384
105532024-01-24T19:09:59.903Z22/30 Write block 410 len 16384 data: 5 5 5 5
105542024-01-24T19:09:59.903Z23/30 Write block 282 len 32768 data: 13 13 9 7 6 7 9 8
105552024-01-24T19:09:59.903Z24/30 Read block 390 len 20480
105562024-01-24T19:09:59.906Z25/30 Write block 534 len 8192 data: 8 7
105572024-01-24T19:09:59.907Z26/30 Read block 185 len 8192
105582024-01-24T19:09:59.907Z27/30 Write block 410 len 16384 data: 6 6 6 6
105592024-01-24T19:09:59.908Z28/30 Write block 208 len 4096 data: 6
105602024-01-24T19:09:59.908Z29/30 Read block 27 len 40960
105612024-01-24T19:09:59.911Z30/30 Read block 53 len 36864
105622024-01-24T19:09:59.912Z----------------------------------------------------------------
105632024-01-24T19:09:59.912Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
105642024-01-24T19:09:59.912ZStates: Active Active Active
105652024-01-24T19:09:59.912ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
105662024-01-24T19:09:59.912Z 1 Acked 1000 Write 6 Done Done Done false
105672024-01-24T19:09:59.912Z 2 Acked 1001 Write 8 Done Done Sent false
105682024-01-24T19:09:59.912Z 3 Acked 1002 Write 10 Done Done Sent false
105692024-01-24T19:09:59.912Z 4 Acked 1003 Write 10 Done Done Sent false
105702024-01-24T19:09:59.912Z 5 Acked 1004 Write 6 Done Done Sent false
105712024-01-24T19:09:59.912Z 6 Acked 1005 Write 10 Done Done Sent false
105722024-01-24T19:09:59.912Z 7 Acked 1006 Read 1 Done Done Sent false
105732024-01-24T19:09:59.912Z 8 Acked 1007 Write 6 Done Done Sent false
105742024-01-24T19:09:59.912Z 9 Acked 1008 Flush 0 Done Done Sent false
105752024-01-24T19:09:59.913Z 10 Acked 1009 Write 6 Done Done Sent false
105762024-01-24T19:09:59.913Z 11 Acked 1010 Read 8 Done Done Sent false
105772024-01-24T19:09:59.913Z 12 Acked 1011 Read 10 Done Done Sent false
105782024-01-24T19:09:59.913Z 13 Acked 1012 Flush 0 Done Done Sent false
105792024-01-24T19:09:59.913Z 14 Acked 1013 Write 6 Done Done Sent false
105802024-01-24T19:09:59.913Z 15 Acked 1014 Write 9 Done Done Sent false
105812024-01-24T19:09:59.914Z 16 Acked 1015 Read 10 Done Done Sent false
105822024-01-24T19:09:59.914Z 17 Acked 1016 Read 3 Done Done Sent false
105832024-01-24T19:09:59.914Z 18 Acked 1017 Read 2 Done Done Sent false
105842024-01-24T19:09:59.914Z 19 Acked 1018 Write 7 Done Done Sent false
105852024-01-24T19:09:59.914Z 20 Acked 1019 Write 4 Done Done Sent false
105862024-01-24T19:09:59.914Z 21 Acked 1020 Read 4 Done Done Sent false
105872024-01-24T19:09:59.914Z 22 Acked 1021 Write 4 Done Done Sent false
105882024-01-24T19:09:59.914Z 23 Acked 1022 Write 8 Done Done Sent false
105892024-01-24T19:09:59.914Z 24 Acked 1023 Read 5 Done Done Sent false
105902024-01-24T19:09:59.914Z 25 Acked 1024 Write 2 Done Done Sent false
105912024-01-24T19:09:59.915Z 26 Acked 1025 Read 2 Done Done Sent false
105922024-01-24T19:09:59.915Z 27 Acked 1026 Write 4 Done Done Sent false
105932024-01-24T19:09:59.916Z 28 Acked 1027 Write 1 Done Done Sent false
105942024-01-24T19:09:59.916Z 29 Acked 1028 Read 10 Done Done Sent false
105952024-01-24T19:09:59.916Z 30 Acked 1029 Read 9 Done Sent Sent false
105962024-01-24T19:09:59.916Z STATES DS:0 DS:1 DS:2 TOTAL
105972024-01-24T19:09:59.916Z New 0 0 0 0
105982024-01-24T19:09:59.916Z Sent 0 1 29 30
105992024-01-24T19:09:59.916Z Done 30 29 1 60
106002024-01-24T19:09:59.916Z Skipped 0 0 0 0
106012024-01-24T19:09:59.916Z Error 0 0 0 0
106022024-01-24T19:09:59.916ZLast Flush: 1012 1012 0
106032024-01-24T19:09:59.916ZDownstairs last five completed:
106042024-01-24T19:09:59.916ZUpstairs last five completed: 30 29 28 27 26
106052024-01-24T19:09:59.922ZWrote out file "/tmp/test_repair_verify.data"
106062024-01-24T19:09:59.922Z
106072024-01-24T19:09:59.929ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106082024-01-24T19:09:59.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106092024-01-24T19:09:59.952Z 0 000-019 92 92 92 157 157 157 T T F <---
106102024-01-24T19:09:59.952Z 1 020-039 96 96 96 166 166 166 F F F
106112024-01-24T19:09:59.952Z 2 040-059 91 91 91 154 154 154 F F F
106122024-01-24T19:09:59.952Z 3 060-079 96 96 96 167 167 167 F F F
106132024-01-24T19:09:59.952Z 4 080-099 95 95 95 164 164 164 F F F
106142024-01-24T19:09:59.956Z 5 100-119 96 96 96 167 167 167 F F F
106152024-01-24T19:09:59.956Z 6 120-139 97 97 97 168 168 168 F F F
106162024-01-24T19:09:59.956Z 7 140-159 87 87 87 144 144 144 F F F
106172024-01-24T19:09:59.956Z 8 160-179 97 97 97 168 168 168 F F F
106182024-01-24T19:09:59.956Z 9 180-199 98 98 92 169 169 155 T T F <---
106192024-01-24T19:09:59.956Z 10 200-219 95 95 95 164 164 164 T T F <---
106202024-01-24T19:09:59.956Z 11 220-239 92 92 92 157 157 157 F F F
106212024-01-24T19:09:59.956Z 12 240-259 97 97 97 168 168 168 F F F
106222024-01-24T19:09:59.957Z 13 260-279 98 98 92 169 169 159 T T F <---
106232024-01-24T19:09:59.957Z 14 280-299 98 98 92 169 169 159 T T F <---
106242024-01-24T19:09:59.957Z 15 300-319 89 89 89 148 148 148 F F F
106252024-01-24T19:09:59.957Z 16 320-339 94 94 94 163 163 163 F F F
106262024-01-24T19:09:59.957Z 17 340-359 96 96 96 167 167 167 F F F
106272024-01-24T19:09:59.957Z 18 360-379 98 98 97 169 169 168 F F F <---
106282024-01-24T19:09:59.957Z 19 380-399 97 97 97 168 168 168 F F F
106292024-01-24T19:09:59.957Z 20 400-419 97 97 97 168 168 168 T T F <---
106302024-01-24T19:09:59.957Z 21 420-439 94 94 94 161 161 161 F F F
106312024-01-24T19:09:59.957Z 22 440-459 92 92 92 155 155 155 F F F
106322024-01-24T19:09:59.957Z 23 460-479 94 94 94 162 162 162 F F F
106332024-01-24T19:09:59.957Z 24 480-499 98 98 93 169 169 160 F F F <---
106342024-01-24T19:09:59.958Z 25 500-519 98 98 96 169 169 167 F F T <---
106352024-01-24T19:09:59.958Z 26 520-539 94 94 94 161 161 161 T T F <---
106362024-01-24T19:09:59.958Z 27 540-559 94 94 94 161 161 161 F F F
106372024-01-24T19:09:59.958Z 28 560-579 98 98 94 170 170 161 F F F <---
106382024-01-24T19:09:59.958Z 29 580-599 90 90 90 149 149 149 F F F
106392024-01-24T19:09:59.958ZMax gen: 98, Max flush: 170
106402024-01-24T19:09:59.958ZError: Difference in extent metadata found!
106412024-01-24T19:09:59.962ZA Difference in extent metadata is expected here
106422024-01-24T19:09:59.962ZOn loop 49
106432024-01-24T19:09:59.962Z
106442024-01-24T19:09:59.962ZVerifying data now
106452024-01-24T19:10:04.986ZNo change after verify
106462024-01-24T19:10:04.986ZLoop: 49 Downstairs dump after verify (and repair):
106472024-01-24T19:10:05.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106482024-01-24T19:10:05.010Z 0 000-019 99 99 99 171 171 171 F F F
106492024-01-24T19:10:05.015Z 1 020-039 96 96 96 166 166 166 F F F
106502024-01-24T19:10:05.015Z 2 040-059 91 91 91 154 154 154 F F F
106512024-01-24T19:10:05.015Z 3 060-079 96 96 96 167 167 167 F F F
106522024-01-24T19:10:05.015Z 4 080-099 95 95 95 164 164 164 F F F
106532024-01-24T19:10:05.015Z 5 100-119 96 96 96 167 167 167 F F F
106542024-01-24T19:10:05.015Z 6 120-139 97 97 97 168 168 168 F F F
106552024-01-24T19:10:05.015Z 7 140-159 87 87 87 144 144 144 F F F
106562024-01-24T19:10:05.015Z 8 160-179 97 97 97 168 168 168 F F F
106572024-01-24T19:10:05.015Z 9 180-199 99 99 99 171 171 171 F F F
106582024-01-24T19:10:05.016Z 10 200-219 99 99 99 171 171 171 F F F
106592024-01-24T19:10:05.016Z 11 220-239 92 92 92 157 157 157 F F F
106602024-01-24T19:10:05.016Z 12 240-259 97 97 97 168 168 168 F F F
106612024-01-24T19:10:05.016Z 13 260-279 99 99 99 171 171 171 F F F
106622024-01-24T19:10:05.016Z 14 280-299 99 99 99 171 171 171 F F F
106632024-01-24T19:10:05.016Z 15 300-319 89 89 89 148 148 148 F F F
106642024-01-24T19:10:05.016Z 16 320-339 94 94 94 163 163 163 F F F
106652024-01-24T19:10:05.016Z 17 340-359 96 96 96 167 167 167 F F F
106662024-01-24T19:10:05.016Z 18 360-379 98 98 98 169 169 169 F F F
106672024-01-24T19:10:05.017Z 19 380-399 97 97 97 168 168 168 F F F
106682024-01-24T19:10:05.017Z 20 400-419 99 99 99 171 171 171 F F F
106692024-01-24T19:10:05.017Z 21 420-439 94 94 94 161 161 161 F F F
106702024-01-24T19:10:05.017Z 22 440-459 92 92 92 155 155 155 F F F
106712024-01-24T19:10:05.017Z 23 460-479 94 94 94 162 162 162 F F F
106722024-01-24T19:10:05.017Z 24 480-499 98 98 98 169 169 169 F F F
106732024-01-24T19:10:05.017Z 25 500-519 98 98 98 169 169 169 F F F
106742024-01-24T19:10:05.017Z 26 520-539 99 99 99 171 171 171 F F F
106752024-01-24T19:10:05.017Z 27 540-559 94 94 94 161 161 161 F F F
106762024-01-24T19:10:05.017Z 28 560-579 98 98 98 170 170 170 F F F
106772024-01-24T19:10:05.017Z 29 580-599 90 90 90 149 149 149 F F F
106782024-01-24T19:10:05.017ZMax gen: 99, Max flush: 171
106792024-01-24T19:10:05.017Z
106802024-01-24T19:10:05.017ZBegin loop 50
106812024-01-24T19:10:05.017ZDownstairs to restart: 1
106822024-01-24T19:10:05.049Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.711080383Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529}
106832024-01-24T19:10:05.049Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71156369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529}
106842024-01-24T19:10:05.050Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.711601259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529}
106852024-01-24T19:10:05.500Z{"msg":"Crucible 804ffa43-37f7-4939-b24f-e0ecd07d3e9f has session id: 8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2","v":0,"name":"crucible","level":30Crucible runtime is spawned
106862024-01-24T19:10:05.501Z,"time":"2024-01-24T19:10:03.711623321Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106872024-01-24T19:10:05.501Z{"msg":"Upstairs opts: Upstairs UUID: 804ffa43-37f7-4939-b24f-e0ecd07d3e9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.711998023Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106882024-01-24T19:10:05.504Z{"msg":"Crucible stats registered with UUID: 804ffa43-37f7-4939-b24f-e0ecd07d3e9f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712076202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106892024-01-24T19:10:05.504Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71214174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529}
106902024-01-24T19:10:05.504Z{"msg":"804ffa43-37f7-4939-b24f-e0ecd07d3e9f active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.7121724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106912024-01-24T19:10:05.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712235912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106922024-01-24T19:10:05.504Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712281303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106932024-01-24T19:10:05.505Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712313322Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106942024-01-24T19:10:05.505Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712350885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106952024-01-24T19:10:05.505Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712392004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106962024-01-24T19:10:05.505Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712459881Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106972024-01-24T19:10:05.505Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712504122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106982024-01-24T19:10:05.505Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712596174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
106992024-01-24T19:10:05.505Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712809673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107002024-01-24T19:10:05.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712889168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107012024-01-24T19:10:05.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.712974052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107022024-01-24T19:10:05.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.713049469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107032024-01-24T19:10:05.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71396649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107042024-01-24T19:10:05.506Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71405425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107052024-01-24T19:10:05.506Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.714103483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107062024-01-24T19:10:05.506Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.714178917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107072024-01-24T19:10:05.506Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.714321873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107082024-01-24T19:10:05.507Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.714390482Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107092024-01-24T19:10:05.507Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.714949082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107102024-01-24T19:10:05.507Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.715342559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107112024-01-24T19:10:05.507Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.715488201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107122024-01-24T19:10:05.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.715935119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107132024-01-24T19:10:05.507Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716027729Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107142024-01-24T19:10:05.507Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716159009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107152024-01-24T19:10:05.507Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71618333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107162024-01-24T19:10:05.507Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716205575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107172024-01-24T19:10:05.507Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716227134Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107182024-01-24T19:10:05.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716247375Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107192024-01-24T19:10:05.508Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716305273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107202024-01-24T19:10:05.508Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 144, 168, 171, 171, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.71639707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107212024-01-24T19:10:05.508Z{"msg":"[0]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 87, 97, 99, 99, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716429129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107222024-01-24T19:10:05.508Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716453301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107232024-01-24T19:10:05.508Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 144, 168, 171, 171, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716477064Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107242024-01-24T19:10:05.509Z{"msg":"[1]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 87, 97, 99, 99, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716499587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107252024-01-24T19:10:05.509Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716522717Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107262024-01-24T19:10:05.509Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 144, 168, 171, 171, 157]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
107272024-01-24T19:10:05.509Z,"time":"2024-01-24T19:10:03.716545811Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107282024-01-24T19:10:05.510Z{"msg":"[2]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 87, 97, 99, 99, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716569529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107292024-01-24T19:10:05.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716592689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107302024-01-24T19:10:05.510Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716639825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107312024-01-24T19:10:05.510Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716665866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107322024-01-24T19:10:05.511Z{"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716687243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107332024-01-24T19:10:05.511Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716729685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107342024-01-24T19:10:05.511Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716755858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107352024-01-24T19:10:05.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716776112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107362024-01-24T19:10:05.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716818655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107372024-01-24T19:10:05.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716844013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107382024-01-24T19:10:05.512Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716866118Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107392024-01-24T19:10:05.512Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716911582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107402024-01-24T19:10:05.512Z{"msg":"804ffa43-37f7-4939-b24f-e0ecd07d3e9f is now active with session: 8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107412024-01-24T19:10:05.512Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716957381Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529,"session_id":"8fde9dc9-207c-4aa6-9fa3-e2276fbf27a2"}
107422024-01-24T19:10:05.512Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:03.716979565Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1529}
107432024-01-24T19:10:05.512ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107442024-01-24T19:10:05.512ZLoading write count information from file "/tmp/test_repair_verify.data"
107452024-01-24T19:10:05.512ZRun Repair workload
107462024-01-24T19:10:05.512Z01/30 Write block 462 len 28672 data: 7 6 6 8 8 10 8
107472024-01-24T19:10:05.512Z02/30 Read block 520 len 40960
107482024-01-24T19:10:05.512Z03/30 Flush
107492024-01-24T19:10:05.512Z04/30 Write block 341 len 12288 data: 7 7 9
107502024-01-24T19:10:05.512Z05/30 Write block 558 len 16384 data: 8 7 5 6
107512024-01-24T19:10:05.512Z06/30 Write block 197 len 20480 data: 7 6 4 4 5
107522024-01-24T19:10:05.512Z07/30 Write block 313 len 32768 data: 8 9 10 12 12 11 11 11
107532024-01-24T19:10:05.512Z08/30 Write block 517 len 16384 data: 13 13 12 10
107542024-01-24T19:10:05.512Z09/30 Read block 567 len 4096
107552024-01-24T19:10:05.512Z10/30 Write block 147 len 20480 data: 8 6 8 8 10
107562024-01-24T19:10:05.512Z11/30 Read block 297 len 16384
107572024-01-24T19:10:05.512Z12/30 Write block 261 len 8192 data: 7 7
107582024-01-24T19:10:05.512Z13/30 Write block 429 len 24576 data: 11 10 11 13 13 13
107592024-01-24T19:10:05.513Z14/30 Read block 55 len 24576
107602024-01-24T19:10:05.513Z15/30 Read block 115 len 28672
107612024-01-24T19:10:05.513Z16/30 Flush
107622024-01-24T19:10:05.513Z17/30 Write block 343 len 28672 data: 10 9 9 6 5 5 7
107632024-01-24T19:10:05.513Z18/30 Write block 488 len 4096 data: 5
107642024-01-24T19:10:05.513Z19/30 Write block 301 len 12288 data: 5 5 4
107652024-01-24T19:10:05.513Z20/30 Write block 192 len 8192 data: 8 9
107662024-01-24T19:10:05.513Z21/30 Read block 207 len 8192
107672024-01-24T19:10:05.513Z22/30 Write block 551 len 12288 data: 7 6 5
107682024-01-24T19:10:05.513Z23/30 Flush
107692024-01-24T19:10:05.513Z24/30 Write block 494 len 28672 data: 8 8 11 11 11 10 7
107702024-01-24T19:10:05.513Z25/30 Write block 590 len 20480 data: 6 8 11 10 9
107712024-01-24T19:10:05.513Z26/30 Write block 154 len 20480 data: 9 7 7 8 8
107722024-01-24T19:10:05.513Z27/30 Read block 528 len 8192
107732024-01-24T19:10:05.513Z28/30 Write block 332 len 40960 data: 10 8 7 8 8 10 10 10 9 8
107742024-01-24T19:10:05.513Z29/30 Read block 176 len 24576
107752024-01-24T19:10:05.513Z30/30 Write block 323 len 28672 data: 10 9 11 7 7 8 9
107762024-01-24T19:10:05.513Z----------------------------------------------------------------
107772024-01-24T19:10:05.513Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:27
107782024-01-24T19:10:05.513ZStates: Active Active Active
107792024-01-24T19:10:05.513ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
107802024-01-24T19:10:05.516Z 4 Acked 1003 Write 3 Done Sent Done false
107812024-01-24T19:10:05.516Z 5 Acked 1004 Write 4 Done Sent Done false
107822024-01-24T19:10:05.516Z 6 Acked 1005 Write 5 Done Sent Done false
107832024-01-24T19:10:05.516Z 7 Acked 1006 Write 8 Done Sent Done false
107842024-01-24T19:10:05.516Z 8 Acked 1007 Write 4 Done Sent Done false
107852024-01-24T19:10:05.522Z 9 Acked 1008 Read 1 Done Sent Done false
107862024-01-24T19:10:05.522Z 10 Acked 1009 Write 5 Done Sent Done false
107872024-01-24T19:10:05.522Z 11 Acked 1010 Read 4 Done Sent Done false
107882024-01-24T19:10:05.522Z 12 Acked 1011 Write 2 Done Sent Done false
107892024-01-24T19:10:05.522Z 13 Acked 1012 Write 6 Done Sent Done false
107902024-01-24T19:10:05.522Z 14 Acked 1013 Read 6 Done Sent Done false
107912024-01-24T19:10:05.522Z 15 Acked 1014 Read 7 Done Sent Done false
107922024-01-24T19:10:05.522Z 16 Acked 1015 Flush 0 Done Sent Done false
107932024-01-24T19:10:05.523Z 17 Acked 1016 Write 7 Done Sent Done false
107942024-01-24T19:10:05.523Z 18 Acked 1017 Write 1 Done Sent Done false
107952024-01-24T19:10:05.523Z 19 Acked 1018 Write 3 Done Sent Done false
107962024-01-24T19:10:05.523Z 20 Acked 1019 Write 2 Done Sent Done false
107972024-01-24T19:10:05.523Z 21 Acked 1020 Read 2 Done Sent Done false
107982024-01-24T19:10:05.523Z 22 Acked 1021 Write 3 Done Sent Done false
107992024-01-24T19:10:05.523Z 23 Acked 1022 Flush 0 Done Sent Done false
108002024-01-24T19:10:05.523Z 24 Acked 1023 Write 7 Done Sent Done false
108012024-01-24T19:10:05.523Z 25 Acked 1024 Write 5 Done Sent Done false
108022024-01-24T19:10:05.523Z 26 Acked 1025 Write 5 Done Sent Done false
108032024-01-24T19:10:05.524Z 27 Acked 1026 Read 2 Done Sent Done false
108042024-01-24T19:10:05.524Z 28 Acked 1027 Write 10 Done Sent Done false
108052024-01-24T19:10:05.524Z 29 Acked 1028 Read 6 Done Sent Done false
108062024-01-24T19:10:05.524Z 30 Acked 1029 Write 7 Sent Sent Sent false
108072024-01-24T19:10:05.524Z STATES DS:0 DS:1 DS:2 TOTAL
108082024-01-24T19:10:05.524Z New 0 0 0 0
108092024-01-24T19:10:05.524Z Sent 1 27 1 29
108102024-01-24T19:10:05.524Z Done 26 0 26 52
108112024-01-24T19:10:05.524Z Skipped 0 0 0 0
108122024-01-24T19:10:05.524Z Error 0 0 0 0
108132024-01-24T19:10:05.525ZLast Flush: 1022 1002 1022
108142024-01-24T19:10:05.525ZDownstairs last five completed: 1002 1001 1000
108152024-01-24T19:10:05.525ZUpstairs last five completed: 30 29 28 27 26
108162024-01-24T19:10:05.525ZWrote out file "/tmp/test_repair_verify.data"
108172024-01-24T19:10:05.525Z
108182024-01-24T19:10:05.525ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108192024-01-24T19:10:05.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108202024-01-24T19:10:05.525Z 0 000-019 99 99 99 171 171 171 F F F
108212024-01-24T19:10:05.525Z 1 020-039 96 96 96 166 166 166 F F F
108222024-01-24T19:10:05.525Z 2 040-059 91 91 91 154 154 154 F F F
108232024-01-24T19:10:05.525Z 3 060-079 96 96 96 167 167 167 F F F
108242024-01-24T19:10:05.525Z 4 080-099 95 95 95 164 164 164 F F F
108252024-01-24T19:10:05.525Z 5 100-119 96 96 96 167 167 167 F F F
108262024-01-24T19:10:05.525Z 6 120-139 97 97 97 168 168 168 F F F
108272024-01-24T19:10:05.525Z 7 140-159 100 87 100 173 144 173 T F T <---
108282024-01-24T19:10:05.525Z 8 160-179 97 97 97 168 168 168 F F F
108292024-01-24T19:10:05.525Z 9 180-199 100 99 100 174 171 174 F F F <---
108302024-01-24T19:10:05.525Z 10 200-219 100 99 100 173 171 173 F F F <---
108312024-01-24T19:10:05.525Z 11 220-239 92 92 92 157 157 157 F F F
108322024-01-24T19:10:05.525Z 12 240-259 97 97 97 168 168 168 F F F
108332024-01-24T19:10:05.525Z 13 260-279 100 99 100 173 171 173 F F F <---
108342024-01-24T19:10:05.525Z 14 280-299 99 99 99 171 171 171 F F F
108352024-01-24T19:10:05.526Z 15 300-319 100 89 100 174 148 174 F F F <---
108362024-01-24T19:10:05.526Z 16 320-339 100 94 100 173 163 173 T F T <---
108372024-01-24T19:10:05.526Z 17 340-359 100 96 100 174 167 174 T F T <---
108382024-01-24T19:10:05.526Z 18 360-379 98 98 98 169 169 169 F F F
108392024-01-24T19:10:05.526Z 19 380-399 97 97 97 168 168 168 F F F
108402024-01-24T19:10:05.526Z 20 400-419 99 99 99 171 171 171 F F F
108412024-01-24T19:10:05.526Z 21 420-439 100 94 100 173 161 173 F F F <---
108422024-01-24T19:10:05.526Z 22 440-459 92 92 92 155 155 155 F F F
108432024-01-24T19:10:05.526Z 23 460-479 100 100 100 172 172 172 F F F
108442024-01-24T19:10:05.526Z 24 480-499 100 98 100 174 169 174 T F T <---
108452024-01-24T19:10:05.526Z 25 500-519 100 98 100 173 169 173 T F T <---
108462024-01-24T19:10:05.526Z 26 520-539 100 99 100 173 171 173 F F F <---
108472024-01-24T19:10:05.526Z 27 540-559 100 94 100 174 161 174 F F F <---
108482024-01-24T19:10:05.526Z 28 560-579 100 98 100 173 170 173 F F F <---
108492024-01-24T19:10:05.526Z 29 580-599 90 90 90 149 149 149 T F T <---
108502024-01-24T19:10:05.526ZMax gen: 100, Max flush: 174
108512024-01-24T19:10:05.526ZError: Difference in extent metadata found!
108522024-01-24T19:10:05.526ZA Difference in extent metadata is expected here
108532024-01-24T19:10:05.526ZOn loop 50
108542024-01-24T19:10:05.526Z
108552024-01-24T19:10:05.526ZVerifying data now
108562024-01-24T19:10:10.745ZNo change after verify
108572024-01-24T19:10:10.961ZLoop: 50 Downstairs dump after verify (and repair):
108582024-01-24T19:10:10.961ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108592024-01-24T19:10:10.961Z 0 000-019 99 99 99 171 171 171 F F F
108602024-01-24T19:10:10.961Z 1 020-039 96 96 96 166 166 166 F F F
108612024-01-24T19:10:10.961Z 2 040-059 91 91 91 154 154 154 F F F
108622024-01-24T19:10:10.961Z 3 060-079 96 96 96 167 167 167 F F F
108632024-01-24T19:10:10.961Z 4 080-099 95 95 95 164 164 164 F F F
108642024-01-24T19:10:10.961Z 5 100-119 96 96 96 167 167 167 F F F
108652024-01-24T19:10:10.961Z 6 120-139 97 97 97 168 168 168 F F F
108662024-01-24T19:10:10.961Z 7 140-159 101 101 101 175 175 175 F F F
108672024-01-24T19:10:10.961Z 8 160-179 97 97 97 168 168 168 F F F
108682024-01-24T19:10:10.964Z 9 180-199 100 100 100 174 174 174 F F F
108692024-01-24T19:10:10.964Z 10 200-219 100 100 100 173 173 173 F F F
108702024-01-24T19:10:10.964Z 11 220-239 92 92 92 157 157 157 F F F
108712024-01-24T19:10:10.964Z 12 240-259 97 97 97 168 168 168 F F F
108722024-01-24T19:10:10.964Z 13 260-279 100 100 100 173 173 173 F F F
108732024-01-24T19:10:10.964Z 14 280-299 99 99 99 171 171 171 F F F
108742024-01-24T19:10:10.964Z 15 300-319 100 100 100 174 174 174 F F F
108752024-01-24T19:10:10.964Z 16 320-339 101 101 101 175 175 175 F F F
108762024-01-24T19:10:10.964Z 17 340-359 101 101 101 175 175 175 F F F
108772024-01-24T19:10:10.964Z 18 360-379 98 98 98 169 169 169 F F F
108782024-01-24T19:10:10.964Z 19 380-399 97 97 97 168 168 168 F F F
108792024-01-24T19:10:10.964Z 20 400-419 99 99 99 171 171 171 F F F
108802024-01-24T19:10:10.964Z 21 420-439 100 100 100 173 173 173 F F F
108812024-01-24T19:10:10.964Z 22 440-459 92 92 92 155 155 155 F F F
108822024-01-24T19:10:10.964Z 23 460-479 100 100 100 172 172 172 F F F
108832024-01-24T19:10:10.964Z 24 480-499 101 101 101 175 175 175 F F F
108842024-01-24T19:10:10.964Z 25 500-519 101 101 101 175 175 175 F F F
108852024-01-24T19:10:10.965Z 26 520-539 100 100 100 173 173 173 F F F
108862024-01-24T19:10:10.965Z 27 540-559 100 100 100 174 174 174 F F F
108872024-01-24T19:10:10.965Z 28 560-579 100 100 100 173 173 173 F F F
108882024-01-24T19:10:10.965Z 29 580-599 101 101 101 175 175 175 F F F
108892024-01-24T19:10:10.965ZMax gen: 101, Max flush: 175
108902024-01-24T19:10:10.965Z
108912024-01-24T19:10:10.965ZBegin loop 51
108922024-01-24T19:10:10.965ZDownstairs to restart: 2
108932024-01-24T19:10:10.966Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.08766601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537}
108942024-01-24T19:10:10.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088171512Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537}
108952024-01-24T19:10:10.967Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088245797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537}
108962024-01-24T19:10:10.967Z{"msg":"Crucible 06c5567a-5d0f-4231-bee5-12771841bc67 has session id: 038fceb1-564a-40f9-9b7d-6a3769780c9d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088310189Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
108972024-01-24T19:10:10.967Z{"msg":"Upstairs opts: Upstairs UUID: 06c5567a-5d0f-4231-bee5-12771841bc67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088394248Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
108982024-01-24T19:10:10.967Z{"msg":"Crucible stats registered with UUID: 06c5567a-5d0f-4231-bee5-12771841bc67","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088427942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
108992024-01-24T19:10:10.967ZCrucible runtime is spawned
109002024-01-24T19:10:10.967Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088681612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537}
109012024-01-24T19:10:10.967Z{"msg":"06c5567a-5d0f-4231-bee5-12771841bc67 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088893383Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109022024-01-24T19:10:10.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.088943257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109032024-01-24T19:10:10.967Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089040963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109042024-01-24T19:10:10.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089135156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109052024-01-24T19:10:10.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089197962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109062024-01-24T19:10:10.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089330174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109072024-01-24T19:10:10.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089373644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109082024-01-24T19:10:10.968Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089576544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109092024-01-24T19:10:10.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089650339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109102024-01-24T19:10:10.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089695073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109112024-01-24T19:10:10.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089769941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109122024-01-24T19:10:10.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089887712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109132024-01-24T19:10:10.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.089952507Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109142024-01-24T19:10:10.981Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.090767993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109152024-01-24T19:10:10.981Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.090910455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109162024-01-24T19:10:10.981Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.090975932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109172024-01-24T19:10:10.981Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.091096543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109182024-01-24T19:10:10.981Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.091492018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109192024-01-24T19:10:10.981Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.09155486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109202024-01-24T19:10:10.981Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.092165283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109212024-01-24T19:10:10.981Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.092258582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109222024-01-24T19:10:10.981Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.092697638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109232024-01-24T19:10:10.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093218118Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109242024-01-24T19:10:10.981Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093306682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109252024-01-24T19:10:10.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093385179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109262024-01-24T19:10:10.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093460804Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109272024-01-24T19:10:10.981Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093559144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109282024-01-24T19:10:10.981Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093621934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109292024-01-24T19:10:10.981Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093676241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109302024-01-24T19:10:10.981Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093723231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109312024-01-24T19:10:10.981Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 175, 168, 174, 173, 157]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
109322024-01-24T19:10:10.981Z2024-01-24T19:10:09.093770559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109332024-01-24T19:10:10.981Z{"msg":"[0]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 101, 97, 100, 100, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093886566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109342024-01-24T19:10:10.981Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093943925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109352024-01-24T19:10:10.982Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 175, 168, 174, 173, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.093991695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109362024-01-24T19:10:10.982Z{"msg":"[1]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 101, 97, 100, 100, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094039343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109372024-01-24T19:10:10.982Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.09406669Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109382024-01-24T19:10:10.982Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 154, 167, 164, 167, 168, 175, 168, 174, 173, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094112652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109392024-01-24T19:10:10.982Z{"msg":"[2]R generation[0..12]: [99, 96, 91, 96, 95, 96, 97, 101, 97, 100, 100, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094172954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109402024-01-24T19:10:10.982Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094224263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109412024-01-24T19:10:10.982Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094273351Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109422024-01-24T19:10:10.982Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094327957Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109432024-01-24T19:10:10.982Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094380217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109442024-01-24T19:10:10.998Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094426395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109452024-01-24T19:10:10.998Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
109462024-01-24T19:10:10.998Z,"time":"2024-01-24T19:10:09.094491141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109472024-01-24T19:10:10.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094592449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109482024-01-24T19:10:10.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094642644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109492024-01-24T19:10:10.998Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094688277Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109502024-01-24T19:10:10.999Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094733221Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109512024-01-24T19:10:10.999Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094776967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109522024-01-24T19:10:10.999Z{"msg":"06c5567a-5d0f-4231-bee5-12771841bc67 is now active with session: 038fceb1-564a-40f9-9b7d-6a3769780c9d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094821837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109532024-01-24T19:10:10.999Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094865092Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537,"session_id":"038fceb1-564a-40f9-9b7d-6a3769780c9d"}
109542024-01-24T19:10:10.999Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:09.094908366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1537}
109552024-01-24T19:10:10.999ZLoading write count information from file "/tmp/test_repair_verify.data"
109562024-01-24T19:10:10.999ZRun Repair workload
109572024-01-24T19:10:11.066Z01/30 Write block 241 len 32768 data: 7 9 11 10 11 13 12 11
109582024-01-24T19:10:11.066Z02/30 Flush
109592024-01-24T19:10:11.066Z03/30 Write block 534 len 24576 data: 9 8 8 9 9 9
109602024-01-24T19:10:11.066Z04/30 Read block 164 len 16384
109612024-01-24T19:10:11.066Z05/30 Write block 207 len 28672 data: 6 7 8 10 8 8 7
109622024-01-24T19:10:11.066Z06/30 Write block 568 len 8192 data: 11 10
109632024-01-24T19:10:11.066Z07/30 Read block 20 len 16384
109642024-01-24T19:10:11.066Z08/30 Write block 349 len 40960 data: 8 6 7 6 6 5 6 8 7 8
109652024-01-24T19:10:11.066Z09/30 Write block 264 len 36864 data: 9 9 8 9 9 9 10 9 11
109662024-01-24T19:10:11.066Z10/30 Write block 296 len 4096 data: 10
109672024-01-24T19:10:11.066Z11/30 Read block 87 len 32768
109682024-01-24T19:10:11.066Z12/30 Read block 470 len 32768
109692024-01-24T19:10:11.066ZError: Difference in extent metadata found!
109702024-01-24T19:10:11.066Z13/30 Read block 433 len 20480
109712024-01-24T19:10:11.067Z14/30 Write block 554 len 32768 data: 5 4 6 8 9 8 6 7
109722024-01-24T19:10:11.067Z15/30 Write block 304 len 28672 data: 3 4 4 5 4 4 5
109732024-01-24T19:10:11.067Z16/30 Read block 356 len 4096
109742024-01-24T19:10:11.067Z17/30 Write block 461 len 4096 data: 7
109752024-01-24T19:10:11.067Z18/30 Write block 489 len 16384 data: 5 6 6 8
109762024-01-24T19:10:11.067Z19/30 Write block 45 len 20480 data: 12 9 7 10 9
109772024-01-24T19:10:11.067Z20/30 Write block 145 len 32768 data: 5 7 9 7 9 9 11 10
109782024-01-24T19:10:11.067Z21/30 Read block 148 len 40960
109792024-01-24T19:10:11.067Z22/30 Flush
109802024-01-24T19:10:11.067Z23/30 Flush
109812024-01-24T19:10:11.067Z24/30 Write block 351 len 12288 data: 8 7 7
109822024-01-24T19:10:11.067Z25/30 Write block 116 len 28672 data: 4 4 6 8 8 8 8
109832024-01-24T19:10:11.077Z26/30 Read block 10 len 28672
109842024-01-24T19:10:11.077Z27/30 Read block 546 len 40960
109852024-01-24T19:10:11.077Z28/30 Write block 162 len 4096 data: 11
109862024-01-24T19:10:11.077Z29/30 Read block 575 len 40960
109872024-01-24T19:10:11.077Z30/30 Write block 97 len 36864 data: 8 7 5 6 4 5 9 10 10
109882024-01-24T19:10:11.077Z----------------------------------------------------------------
109892024-01-24T19:10:11.077Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
109902024-01-24T19:10:11.077ZStates: Active Active Active
109912024-01-24T19:10:11.077ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109922024-01-24T19:10:11.077Z 1 Acked 1000 Write 8 Done Done Sent false
109932024-01-24T19:10:11.077Z 2 Acked 1001 Flush 0 Done Done Sent false
109942024-01-24T19:10:11.077Z 3 Acked 1002 Write 6 Done Done Sent false
109952024-01-24T19:10:11.078Z 4 Acked 1003 Read 4 Done Done Sent false
109962024-01-24T19:10:11.078Z 5 Acked 1004 Write 7 Done Done Sent false
109972024-01-24T19:10:11.078Z 6 Acked 1005 Write 2 Done Done Sent false
109982024-01-24T19:10:11.078Z 7 Acked 1006 Read 4 Done Done Sent false
109992024-01-24T19:10:11.078Z 8 Acked 1007 Write 10 Done Done Sent false
110002024-01-24T19:10:11.078Z 9 Acked 1008 Write 9 Done Done Sent false
110012024-01-24T19:10:11.078Z 10 Acked 1009 Write 1 Done Done Sent false
110022024-01-24T19:10:11.078Z 11 Acked 1010 Read 8 Done Done Sent false
110032024-01-24T19:10:11.078Z 12 Acked 1011 Read 8 Done Done Sent false
110042024-01-24T19:10:11.078Z 13 Acked 1012 Read 5 Done Done Sent false
110052024-01-24T19:10:11.078Z 14 Acked 1013 Write 8 Done Done Sent false
110062024-01-24T19:10:11.078Z 15 Acked 1014 Write 7 Done Done Sent false
110072024-01-24T19:10:11.078Z 16 Acked 1015 Read 1 Done Done Sent false
110082024-01-24T19:10:11.078Z 17 Acked 1016 Write 1 Done Done Sent false
110092024-01-24T19:10:11.078Z 18 Acked 1017 Write 4 Done Done Sent false
110102024-01-24T19:10:11.078Z 19 Acked 1018 Write 5 Done Done Sent false
110112024-01-24T19:10:11.078Z 20 Acked 1019 Write 8 Done Done Sent false
110122024-01-24T19:10:11.078Z 21 Acked 1020 Read 10 Done Done Sent false
110132024-01-24T19:10:11.078Z 22 Acked 1021 Flush 0 Done Done Sent false
110142024-01-24T19:10:11.078Z 23 Acked 1022 Flush 0 Done Done Sent false
110152024-01-24T19:10:11.078Z 24 Acked 1023 Write 3 Done Done Sent false
110162024-01-24T19:10:11.078Z 25 Acked 1024 Write 7 Done Done Sent false
110172024-01-24T19:10:11.079Z 26 Acked 1025 Read 7 Done Done Sent false
110182024-01-24T19:10:11.079Z 27 Acked 1026 Read 10 Done Done Sent false
110192024-01-24T19:10:11.079Z 28 Acked 1027 Write 1 Done Done Sent false
110202024-01-24T19:10:11.079Z 29 Acked 1028 Read 10 Done Done Sent false
110212024-01-24T19:10:11.079Z 30 Acked 1029 Write 9 Sent Sent Sent false
110222024-01-24T19:10:11.079Z STATES DS:0 DS:1 DS:2 TOTAL
110232024-01-24T19:10:11.079Z New 0 0 0 0
110242024-01-24T19:10:11.079Z Sent 1 1 30 32
110252024-01-24T19:10:11.079Z Done 29 29 0 58
110262024-01-24T19:10:11.079Z Skipped 0 0 0 0
110272024-01-24T19:10:11.079Z Error 0 0 0 0
110282024-01-24T19:10:11.079ZLast Flush: 1022 1022 0
110292024-01-24T19:10:11.079ZDownstairs last five completed:
110302024-01-24T19:10:11.079ZUpstairs last five completed: 30 29 28 27 26
110312024-01-24T19:10:11.089ZWrote out file "/tmp/test_repair_verify.data"
110322024-01-24T19:10:11.089Z
110332024-01-24T19:10:11.089ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110342024-01-24T19:10:11.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110352024-01-24T19:10:11.089Z 0 000-019 99 99 99 171 171 171 F F F
110362024-01-24T19:10:11.089Z 1 020-039 96 96 96 166 166 166 F F F
110372024-01-24T19:10:11.089Z 2 040-059 102 102 91 177 177 154 F F F <---
110382024-01-24T19:10:11.089Z 3 060-079 96 96 96 167 167 167 F F F
110392024-01-24T19:10:11.089Z 4 080-099 95 95 95 164 164 164 T T F <---
110402024-01-24T19:10:11.089Z 5 100-119 96 96 96 167 167 167 T T F <---
110412024-01-24T19:10:11.090Z 6 120-139 97 97 97 168 168 168 T T F <---
110422024-01-24T19:10:11.090Z 7 140-159 102 102 101 177 177 175 F F F <---
110432024-01-24T19:10:11.090Z 8 160-179 97 97 97 168 168 168 T T F <---
110442024-01-24T19:10:11.090Z 9 180-199 100 100 100 174 174 174 F F F
110452024-01-24T19:10:11.090Z 10 200-219 102 102 100 177 177 173 F F F <---
110462024-01-24T19:10:11.090Z 11 220-239 92 92 92 157 157 157 F F F
110472024-01-24T19:10:11.091Z 12 240-259 102 102 97 176 176 168 F F F <---
110482024-01-24T19:10:11.091Z 13 260-279 102 102 100 177 177 173 F F F <---
110492024-01-24T19:10:11.091Z 14 280-299 102 102 99 177 177 171 F F F <---
110502024-01-24T19:10:11.091Z 15 300-319 102 102 100 177 177 174 F F F <---
110512024-01-24T19:10:11.091Z 16 320-339 101 101 101 175 175 175 F F F
110522024-01-24T19:10:11.091Z 17 340-359 102 102 101 177 177 175 T T F <---
110532024-01-24T19:10:11.091Z 18 360-379 98 98 98 169 169 169 F F F
110542024-01-24T19:10:11.091Z 19 380-399 97 97 97 168 168 168 F F F
110552024-01-24T19:10:11.137Z 20 400-419 99 99 99 171 171 171 F F F
110562024-01-24T19:10:11.137Z 21 420-439 100 100 100 173 173 173 F F F
110572024-01-24T19:10:11.162Z 22 440-459 92 92 92 155 155 155 F F F
110582024-01-24T19:10:11.163Z 23 460-479 102 102 100 177 177 172 F F F <---
110592024-01-24T19:10:11.163Z 24 480-499 102 102 101 177 177 175 F F F <---
110602024-01-24T19:10:11.163Z 25 500-519 101 101 101 175 175 175 F F F
110612024-01-24T19:10:11.163Z 26 520-539 102 102 100 177 177 173 F F F <---
110622024-01-24T19:10:11.163Z 27 540-559 102 102 100 177 177 174 F F F <---
110632024-01-24T19:10:11.164Z 28 560-579 102 102 100 177 177 173 F F F <---
110642024-01-24T19:10:11.164Z 29 580-599 101 101 101 175 175 175 F F F
110652024-01-24T19:10:11.164ZMax gen: 102, Max flush: 177
110662024-01-24T19:10:11.164ZA Difference in extent metadata is expected here
110672024-01-24T19:10:11.165ZOn loop 51
110682024-01-24T19:10:11.165Z
110692024-01-24T19:10:11.175ZVerifying data now
110702024-01-24T19:10:15.981ZNo change after verify
110712024-01-24T19:10:15.981ZLoop: 51 Downstairs dump after verify (and repair):
110722024-01-24T19:10:16.012ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110732024-01-24T19:10:16.012Z 0 000-019 99 99 99 171 171 171 F F F
110742024-01-24T19:10:16.012Z 1 020-039 96 96 96 166 166 166 F F F
110752024-01-24T19:10:16.012Z 2 040-059 102 102 102 177 177 177 F F F
110762024-01-24T19:10:16.013Z 3 060-079 96 96 96 167 167 167 F F F
110772024-01-24T19:10:16.013Z 4 080-099 103 103 103 178 178 178 F F F
110782024-01-24T19:10:16.013Z 5 100-119 103 103 103 178 178 178 F F F
110792024-01-24T19:10:16.013Z 6 120-139 103 103 103 178 178 178 F F F
110802024-01-24T19:10:16.013Z 7 140-159 102 102 102 177 177 177 F F F
110812024-01-24T19:10:16.013Z 8 160-179 103 103 103 178 178 178 F F F
110822024-01-24T19:10:16.013Z 9 180-199 100 100 100 174 174 174 F F F
110832024-01-24T19:10:16.013Z 10 200-219 102 102 102 177 177 177 F F F
110842024-01-24T19:10:16.014Z 11 220-239 92 92 92 157 157 157 F F F
110852024-01-24T19:10:16.014Z 12 240-259 102 102 102 176 176 176 F F F
110862024-01-24T19:10:16.014Z 13 260-279 102 102 102 177 177 177 F F F
110872024-01-24T19:10:16.014Z 14 280-299 102 102 102 177 177 177 F F F
110882024-01-24T19:10:16.014Z 15 300-319 102 102 102 177 177 177 F F F
110892024-01-24T19:10:16.014Z 16 320-339 101 101 101 175 175 175 F F F
110902024-01-24T19:10:16.014Z 17 340-359 103 103 103 178 178 178 F F F
110912024-01-24T19:10:16.015Z 18 360-379 98 98 98 169 169 169 F F F
110922024-01-24T19:10:16.015Z 19 380-399 97 97 97 168 168 168 F F F
110932024-01-24T19:10:16.015Z 20 400-419 99 99 99 171 171 171 F F F
110942024-01-24T19:10:16.015Z 21 420-439 100 100 100 173 173 173 F F F
110952024-01-24T19:10:16.015Z 22 440-459 92 92 92 155 155 155 F F F
110962024-01-24T19:10:16.015Z 23 460-479 102 102 102 177 177 177 F F F
110972024-01-24T19:10:16.016Z 24 480-499 102 102 102 177 177 177 F F F
110982024-01-24T19:10:16.016Z 25 500-519 101 101 101 175 175 175 F F F
110992024-01-24T19:10:16.016Z 26 520-539 102 102 102 177 177 177 F F F
111002024-01-24T19:10:16.016Z 27 540-559 102 102 102 177 177 177 F F F
111012024-01-24T19:10:16.017Z 28 560-579 102 102 102 177 177 177 F F F
111022024-01-24T19:10:16.017Z 29 580-599 101 101 101 175 175 175 F F F
111032024-01-24T19:10:16.017ZMax gen: 103, Max flush: 178
111042024-01-24T19:10:16.017Z
111052024-01-24T19:10:16.017ZBegin loop 52
111062024-01-24T19:10:16.017ZDownstairs to restart: 2
111072024-01-24T19:10:16.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.698629764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545}
111082024-01-24T19:10:16.063Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699156149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545}
111092024-01-24T19:10:16.063Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699208322Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545}
111102024-01-24T19:10:16.063Z{"msg":"Crucible b1f49fb4-50a3-42b7-86c1-7f9fa190566d has session id: d894feeb-68aa-4bc9-a221-19f392fe4488","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699233196Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111112024-01-24T19:10:16.063Z{"msg":"Upstairs opts: Upstairs UUID: b1f49fb4-50a3-42b7-86c1-7f9fa190566d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.69931404Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111122024-01-24T19:10:16.065Z{"msg":"Crucible stats registered with UUID: b1f49fb4-50a3-42b7-86c1-7f9fa190566d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699357621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111132024-01-24T19:10:16.065ZCrucible runtime is spawned
111142024-01-24T19:10:16.066Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699556954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545}
111152024-01-24T19:10:16.066Z{"msg":"b1f49fb4-50a3-42b7-86c1-7f9fa190566d active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699856297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111162024-01-24T19:10:16.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.699931665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111172024-01-24T19:10:16.066Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700054075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111182024-01-24T19:10:16.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700117406Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111192024-01-24T19:10:16.066Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700162394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111202024-01-24T19:10:16.066Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700226775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111212024-01-24T19:10:16.066Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700270621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111222024-01-24T19:10:16.066Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700364614Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111232024-01-24T19:10:16.066Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700440218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111242024-01-24T19:10:16.066Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700537787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111252024-01-24T19:10:16.067Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700584536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111262024-01-24T19:10:16.067Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700660109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111272024-01-24T19:10:16.067Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.700710733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111282024-01-24T19:10:16.067Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.701457997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111292024-01-24T19:10:16.067Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.701573836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111302024-01-24T19:10:16.067Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.701706617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111312024-01-24T19:10:16.068Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.701840031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111322024-01-24T19:10:16.069Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.702361313Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111332024-01-24T19:10:16.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70263999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111342024-01-24T19:10:16.069Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.702729633Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111352024-01-24T19:10:16.069Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70284279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111362024-01-24T19:10:16.069Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.703339456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111372024-01-24T19:10:16.070Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.703442426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111382024-01-24T19:10:16.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.703488192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111392024-01-24T19:10:16.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70366962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111402024-01-24T19:10:16.070Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70373452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111412024-01-24T19:10:16.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.703762466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111422024-01-24T19:10:16.071Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70380705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111432024-01-24T19:10:16.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704522701Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111442024-01-24T19:10:16.072Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704617183Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111452024-01-24T19:10:16.072Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 177, 167, 178, 178, 178, 177, 178, 174, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704666263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111462024-01-24T19:10:16.072Z{"msg":"[0]R generation[0..12]: [99, 96, 102, 96, 103, 103, 103, 102, 103, 100, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704739472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111472024-01-24T19:10:16.072Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704783784Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111482024-01-24T19:10:16.072Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 177, 167, 178, 178, 178, 177, 178, 174, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704855168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111492024-01-24T19:10:16.072Z{"msg":"[1]R generation[0..12]: [99, 96, 102, 96, 103, 103, 103, 102, 103, 100, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704902763Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO
111502024-01-24T19:10:16.073Z"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111512024-01-24T19:10:16.073Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.704952246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111522024-01-24T19:10:16.075Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 177, 167, 178, 178, 178, 177, 178, 174, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705032444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111532024-01-24T19:10:16.075Z{"msg":"[2]R generation[0..12]: [99, 96, 102, 96, 103, 103, 103, 102, 103, 100, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705110094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111542024-01-24T19:10:16.075Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705185457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111552024-01-24T19:10:16.075Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.70525809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111562024-01-24T19:10:16.075Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705337658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111572024-01-24T19:10:16.075Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111582024-01-24T19:10:16.075Z,"time":"2024-01-24T19:10:14.70541589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111592024-01-24T19:10:16.076Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705463771Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111602024-01-24T19:10:16.076Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705527065Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111612024-01-24T19:10:16.076Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705552997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111622024-01-24T19:10:16.076Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705597019Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111632024-01-24T19:10:16.076Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705621812Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111642024-01-24T19:10:16.076Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705664335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111652024-01-24T19:10:16.077Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705688159Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111662024-01-24T19:10:16.077Z{"msg":"b1f49fb4-50a3-42b7-86c1-7f9fa190566d is now active with session: d894feeb-68aa-4bc9-a221-19f392fe4488","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705730287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111672024-01-24T19:10:16.077Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705753394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545,"session_id":"d894feeb-68aa-4bc9-a221-19f392fe4488"}
111682024-01-24T19:10:16.077Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:14.705794419Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1545}
111692024-01-24T19:10:16.077ZLoading write count information from file "/tmp/test_repair_verify.data"
111702024-01-24T19:10:16.077ZRun Repair workload
111712024-01-24T19:10:16.078Z01/30 Write block 505 len 16384 data: 7 7 9 8
111722024-01-24T19:10:16.078Z02/30 Write block 410 len 8192 data: 7 7
111732024-01-24T19:10:16.078Z03/30 Read block 234 len 16384
111742024-01-24T19:10:16.078Z04/30 Write block 361 len 20480 data: 11 10 9 11 11
111752024-01-24T19:10:16.078Z05/30 Read block 60 len 36864
111762024-01-24T19:10:16.078Z06/30 Write block 388 len 20480 data: 6 6 5 7 7
111772024-01-24T19:10:16.084Z07/30 Write block 354 len 24576 data: 6 7 9 8 9 8
111782024-01-24T19:10:16.084Z08/30 Read block 353 len 16384
111792024-01-24T19:10:16.084Z09/30 Write block 243 len 32768 data: 12 11 12 14 13 12 12 10
111802024-01-24T19:10:16.084Z10/30 Read block 61 len 40960
111812024-01-24T19:10:16.084Z11/30 Read block 266 len 36864
111822024-01-24T19:10:16.084Z12/30 Write block 185 len 8192 data: 10 9
111832024-01-24T19:10:16.084Z13/30 Read block 168 len 32768
111842024-01-24T19:10:16.090Z14/30 Write block 528 len 16384 data: 12 11 10 8
111852024-01-24T19:10:16.090Z15/30 Read block 255 len 16384
111862024-01-24T19:10:16.090Z16/30 Read block 224 len 20480
111872024-01-24T19:10:16.090Z17/30 Read block 425 len 36864
111882024-01-24T19:10:16.091Z18/30 Read block 434 len 36864
111892024-01-24T19:10:16.091Z19/30 Write block 544 len 32768 data: 12 10 9 10 8 7 7 8
111902024-01-24T19:10:16.091Z20/30 Read block 30 len 36864
111912024-01-24T19:10:16.091Z21/30 Read block 393 len 20480
111922024-01-24T19:10:16.091Z22/30 Write block 60 len 40960 data: 7 7 6 7 9 7 4 4 3 3
111932024-01-24T19:10:16.091Z23/30 Read block 399 len 4096
111942024-01-24T19:10:16.091Z24/30 Read block 402 len 16384
111952024-01-24T19:10:16.091Z25/30 Write block 49 len 40960 data: 10 12 9 8 8 9 9 10 10 9
111962024-01-24T19:10:16.091Z26/30 Read block 78 len 8192
111972024-01-24T19:10:16.091Z27/30 Write block 171 len 36864 data: 10 10 8 10 9 9 7 7 6
111982024-01-24T19:10:16.092Z28/30 Read block 123 len 24576
111992024-01-24T19:10:16.092Z29/30 Read block 85 len 32768
112002024-01-24T19:10:16.092Z30/30 Read block 71 len 8192
112012024-01-24T19:10:16.092Z----------------------------------------------------------------
112022024-01-24T19:10:16.092Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112032024-01-24T19:10:16.092ZStates: Active Active Active
112042024-01-24T19:10:16.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112052024-01-24T19:10:16.093Z 1 Acked 1000 Write 4 Done Done Sent false
112062024-01-24T19:10:16.093Z 2 Acked 1001 Write 2 Done Done Sent false
112072024-01-24T19:10:16.093Z 3 Acked 1002 Read 4 Done Done Sent false
112082024-01-24T19:10:16.093Z 4 Acked 1003 Write 5 Done Done Sent false
112092024-01-24T19:10:16.093Z 5 Acked 1004 Read 9 Done Done Sent false
112102024-01-24T19:10:16.093Z 6 Acked 1005 Write 5 Done Done Sent false
112112024-01-24T19:10:16.093Z 7 Acked 1006 Write 6 Done Done Sent false
112122024-01-24T19:10:16.093Z 8 Acked 1007 Read 4 Done Done Sent false
112132024-01-24T19:10:16.093Z 9 Acked 1008 Write 8 Done Done Sent false
112142024-01-24T19:10:16.093Z 10 Acked 1009 Read 10 Done Done Sent false
112152024-01-24T19:10:16.094Z 11 Acked 1010 Read 9 Done Done Sent false
112162024-01-24T19:10:16.094Z 12 Acked 1011 Write 2 Done Done Sent false
112172024-01-24T19:10:16.094Z 13 Acked 1012 Read 8 Done Done Sent false
112182024-01-24T19:10:16.094Z 14 Acked 1013 Write 4 Done Done Sent false
112192024-01-24T19:10:16.094Z 15 Acked 1014 Read 4 Done Done Sent false
112202024-01-24T19:10:16.094Z 16 Acked 1015 Read 5 Done Done Sent false
112212024-01-24T19:10:16.094Z 17 Acked 1016 Read 9 Done Done Sent false
112222024-01-24T19:10:16.094Z 18 Acked 1017 Read 9 Done Done Sent false
112232024-01-24T19:10:16.094Z 19 Acked 1018 Write 8 Done Done Sent false
112242024-01-24T19:10:16.094Z 20 Acked 1019 Read 9 Done Done Sent false
112252024-01-24T19:10:16.094Z 21 Acked 1020 Read 5 Done Done Sent false
112262024-01-24T19:10:16.095Z 22 Acked 1021 Write 10 Done Done Sent false
112272024-01-24T19:10:16.095Z 23 Acked 1022 Read 1 Done Done Sent false
112282024-01-24T19:10:16.095Z 24 Acked 1023 Read 4 Done Done Sent false
112292024-01-24T19:10:16.095Z 25 Acked 1024 Write 10 Done Done Sent false
112302024-01-24T19:10:16.095Z 26 Acked 1025 Read 2 Done Done Sent false
112312024-01-24T19:10:16.095Z 27 Acked 1026 Write 9 Done Done Sent false
112322024-01-24T19:10:16.095Z 28 Acked 1027 Read 6 Done Done Sent false
112332024-01-24T19:10:16.095Z 29 Acked 1028 Read 8 Done Done Sent false
112342024-01-24T19:10:16.095Z 30 Acked 1029 Read 2 Done Sent Sent false
112352024-01-24T19:10:16.095Z STATES DS:0 DS:1 DS:2 TOTAL
112362024-01-24T19:10:16.095Z New 0 0 0 0
112372024-01-24T19:10:16.095Z Sent 0 1 30 31
112382024-01-24T19:10:16.096Z Done 30 29 0 59
112392024-01-24T19:10:16.096Z Skipped 0 0 0 0
112402024-01-24T19:10:16.096Z Error 0 0 0 0
112412024-01-24T19:10:16.096ZLast Flush: 0 0 0
112422024-01-24T19:10:16.096ZDownstairs last five completed:
112432024-01-24T19:10:16.096ZUpstairs last five completed: 30 29 28 27 26
112442024-01-24T19:10:16.096ZWrote out file "/tmp/test_repair_verify.data"
112452024-01-24T19:10:16.100Z
112462024-01-24T19:10:16.100ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112472024-01-24T19:10:16.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112482024-01-24T19:10:16.129Z 0 000-019 99 99 99 171 171 171 F F F
112492024-01-24T19:10:16.130Z 1 020-039 96 96 96 166 166 166 F F F
112502024-01-24T19:10:16.130Z 2 040-059 102 102 102 177 177 177 T T F <---
112512024-01-24T19:10:16.130Z 3 060-079 96 96 96 167 167 167 T T F <---
112522024-01-24T19:10:16.135Z 4 080-099 103 103 103 178 178 178 F F F
112532024-01-24T19:10:16.135Z 5 100-119 103 103 103 178 178 178 F F F
112542024-01-24T19:10:16.135Z 6 120-139 103 103 103 178 178 178 F F F
112552024-01-24T19:10:16.135Z 7 140-159 102 102 102 177 177 177 F F F
112562024-01-24T19:10:16.136Z 8 160-179 103 103 103 178 178 178 T T F <---
112572024-01-24T19:10:16.136Z 9 180-199 100 100 100 174 174 174 T T F <---
112582024-01-24T19:10:16.136Z 10 200-219 102 102 102 177 177 177 F F F
112592024-01-24T19:10:16.136Z 11 220-239 92 92 92 157 157 157 F F F
112602024-01-24T19:10:16.136Z 12 240-259 102 102 102 176 176 176 T T F <---
112612024-01-24T19:10:16.136Z 13 260-279 102 102 102 177 177 177 F F F
112622024-01-24T19:10:16.136Z 14 280-299 102 102 102 177 177 177 F F F
112632024-01-24T19:10:16.136Z 15 300-319 102 102 102 177 177 177 F F F
112642024-01-24T19:10:16.136Z 16 320-339 101 101 101 175 175 175 F F F
112652024-01-24T19:10:16.136Z 17 340-359 103 103 103 178 178 178 T T F <---
112662024-01-24T19:10:16.136Z 18 360-379 98 98 98 169 169 169 T T F <---
112672024-01-24T19:10:16.136Z 19 380-399 97 97 97 168 168 168 T T F <---
112682024-01-24T19:10:16.136Z 20 400-419 99 99 99 171 171 171 T T F <---
112692024-01-24T19:10:16.136Z 21 420-439 100 100 100 173 173 173 F F F
112702024-01-24T19:10:16.137Z 22 440-459 92 92 92 155 155 155 F F F
112712024-01-24T19:10:16.137Z 23 460-479 102 102 102 177 177 177 F F F
112722024-01-24T19:10:16.137Z 24 480-499 102 102 102 177 177 177 F F F
112732024-01-24T19:10:16.137Z 25 500-519 101 101 101 175 175 175 T T F <---
112742024-01-24T19:10:16.137Z 26 520-539 102 102 102 177 177 177 T T F <---
112752024-01-24T19:10:16.137Z 27 540-559 102 102 102 177 177 177 T T F <---
112762024-01-24T19:10:16.137Z 28 560-579 102 102 102 177 177 177 F F F
112772024-01-24T19:10:16.137Z 29 580-599 101 101 101 175 175 175 F F F
112782024-01-24T19:10:16.137ZMax gen: 103, Max flush: 178
112792024-01-24T19:10:16.137ZError: Difference in extent metadata found!
112802024-01-24T19:10:16.137ZA Difference in extent metadata is expected here
112812024-01-24T19:10:16.137ZOn loop 52
112822024-01-24T19:10:16.138Z
112832024-01-24T19:10:16.144ZVerifying data now
112842024-01-24T19:10:21.230ZNo change after verify
112852024-01-24T19:10:21.230ZLoop: 52 Downstairs dump after verify (and repair):
112862024-01-24T19:10:21.255ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112872024-01-24T19:10:21.255Z 0 000-019 99 99 99 171 171 171 F F F
112882024-01-24T19:10:21.260Z 1 020-039 96 96 96 166 166 166 F F F
112892024-01-24T19:10:21.260Z 2 040-059 104 104 104 179 179 179 F F F
112902024-01-24T19:10:21.260Z 3 060-079 104 104 104 179 179 179 F F F
112912024-01-24T19:10:21.260Z 4 080-099 103 103 103 178 178 178 F F F
112922024-01-24T19:10:21.260Z 5 100-119 103 103 103 178 178 178 F F F
112932024-01-24T19:10:21.260Z 6 120-139 103 103 103 178 178 178 F F F
112942024-01-24T19:10:21.260Z 7 140-159 102 102 102 177 177 177 F F F
112952024-01-24T19:10:21.260Z 8 160-179 104 104 104 179 179 179 F F F
112962024-01-24T19:10:21.260Z 9 180-199 104 104 104 179 179 179 F F F
112972024-01-24T19:10:21.261Z 10 200-219 102 102 102 177 177 177 F F F
112982024-01-24T19:10:21.261Z 11 220-239 92 92 92 157 157 157 F F F
112992024-01-24T19:10:21.262Z 12 240-259 104 104 104 179 179 179 F F F
113002024-01-24T19:10:21.262Z 13 260-279 102 102 102 177 177 177 F F F
113012024-01-24T19:10:21.262Z 14 280-299 102 102 102 177 177 177 F F F
113022024-01-24T19:10:21.262Z 15 300-319 102 102 102 177 177 177 F F F
113032024-01-24T19:10:21.263Z 16 320-339 101 101 101 175 175 175 F F F
113042024-01-24T19:10:21.263Z 17 340-359 104 104 104 179 179 179 F F F
113052024-01-24T19:10:21.263Z 18 360-379 104 104 104 179 179 179 F F F
113062024-01-24T19:10:21.263Z 19 380-399 104 104 104 179 179 179 F F F
113072024-01-24T19:10:21.263Z 20 400-419 104 104 104 179 179 179 F F F
113082024-01-24T19:10:21.263Z 21 420-439 100 100 100 173 173 173 F F F
113092024-01-24T19:10:21.263Z 22 440-459 92 92 92 155 155 155 F F F
113102024-01-24T19:10:21.263Z 23 460-479 102 102 102 177 177 177 F F F
113112024-01-24T19:10:21.263Z 24 480-499 102 102 102 177 177 177 F F F
113122024-01-24T19:10:21.263Z 25 500-519 104 104 104 179 179 179 F F F
113132024-01-24T19:10:21.264Z 26 520-539 104 104 104 179 179 179 F F F
113142024-01-24T19:10:21.265Z 27 540-559 104 104 104 179 179 179 F F F
113152024-01-24T19:10:21.265Z 28 560-579 102 102 102 177 177 177 F F F
113162024-01-24T19:10:21.265Z 29 580-599 101 101 101 175 175 175 F F F
113172024-01-24T19:10:21.265ZMax gen: 104, Max flush: 179
113182024-01-24T19:10:21.265Z
113192024-01-24T19:10:21.265ZBegin loop 53
113202024-01-24T19:10:21.265ZDownstairs to restart: 0
113212024-01-24T19:10:21.283Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.950436219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553}
113222024-01-24T19:10:21.283Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.950855856Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553}
113232024-01-24T19:10:21.283Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.950908332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553}
113242024-01-24T19:10:21.288Z{"msg":"Crucible 68c39255-926d-41c8-b7dd-3f843754f564 has session id: aff31a07-ab42-42fb-aa3b-06baf944af31","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95093346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113252024-01-24T19:10:21.288Z{"msg":"Upstairs opts: Upstairs UUID: 68c39255-926d-41c8-b7dd-3f843754f564, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951024079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113262024-01-24T19:10:21.288Z{"msg":"Crucible stats registered with UUID: 68c39255-926d-41c8-b7dd-3f843754f564","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95106934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113272024-01-24T19:10:21.288ZCrucible runtime is spawned
113282024-01-24T19:10:21.289Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951410053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553}
113292024-01-24T19:10:21.290Z{"msg":"68c39255-926d-41c8-b7dd-3f843754f564 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951572596Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113302024-01-24T19:10:21.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951655463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113312024-01-24T19:10:21.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951685736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113322024-01-24T19:10:21.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951807546Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113332024-01-24T19:10:21.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951843901Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113342024-01-24T19:10:21.290Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.951937876Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113352024-01-24T19:10:21.290Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95198237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113362024-01-24T19:10:21.290Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952020363Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113372024-01-24T19:10:21.290Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952063492Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113382024-01-24T19:10:21.290Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952288622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113392024-01-24T19:10:21.291Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952338675Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113402024-01-24T19:10:21.292Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952362665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113412024-01-24T19:10:21.292Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.952385017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113422024-01-24T19:10:21.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.953560336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113432024-01-24T19:10:21.292Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.953631362Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113442024-01-24T19:10:21.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.953659092Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113452024-01-24T19:10:21.292Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.9536826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113462024-01-24T19:10:21.292Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.954113093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113472024-01-24T19:10:21.292Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.954185341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113482024-01-24T19:10:21.292Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.954797806Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113492024-01-24T19:10:21.293Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.954932181Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113502024-01-24T19:10:21.299Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.955340582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113512024-01-24T19:10:21.299Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.955672293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113522024-01-24T19:10:21.299Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.955808642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113532024-01-24T19:10:21.300Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.955868985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113542024-01-24T19:10:21.300Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.955977238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113552024-01-24T19:10:21.300Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956043615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113562024-01-24T19:10:21.300Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956071712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113572024-01-24T19:10:21.300Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956092917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113582024-01-24T19:10:21.300Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956136526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113592024-01-24T19:10:21.301Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 179, 179, 178, 178, 178, 177, 179, 179, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956163967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113602024-01-24T19:10:21.301Z{"msg":"[0]R generation[0..12]: [99, 96, 104, 104, 103, 103, 103, 102, 104, 104, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956187787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113612024-01-24T19:10:21.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956213219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113622024-01-24T19:10:21.301Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 179, 179, 178, 178, 178, 177, 179, 179, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95623678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113632024-01-24T19:10:21.301Z{"msg":"[1]R generation[0..12]: [99, 96, 104, 104, 103, 103, 103, 102, 104, 104, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95628132Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113642024-01-24T19:10:21.301Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956308544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113652024-01-24T19:10:21.301Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 179, 179, 178, 178, 178, 177, 179, 179, 177, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956353103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113662024-01-24T19:10:21.301Z{"msg":"[2]R generation[0..12]: [99, 96, 104, 104, 103, 103, 103, 102, 104, 104, 102, 92]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.95637964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113672024-01-24T19:10:21.301Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
113682024-01-24T19:10:21.303Zv":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956445394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113692024-01-24T19:10:21.303Z{"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956519939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113702024-01-24T19:10:21.303Z{"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956595155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113712024-01-24T19:10:21.303Z{"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956660055Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113722024-01-24T19:10:21.303Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956687167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113732024-01-24T19:10:21.303Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956738277Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113742024-01-24T19:10:21.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956775458Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113752024-01-24T19:10:21.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956817299Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113762024-01-24T19:10:21.303Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956868691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113772024-01-24T19:10:21.303Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956895047Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113782024-01-24T19:10:21.303Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956962731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113792024-01-24T19:10:21.305Z{"msg":"68c39255-926d-41c8-b7dd-3f843754f564 is now active with session: aff31a07-ab42-42fb-aa3b-06baf944af31","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.956993732Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113802024-01-24T19:10:21.305Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.957048799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553,"session_id":"aff31a07-ab42-42fb-aa3b-06baf944af31"}
113812024-01-24T19:10:21.305Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:19.957115431Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1553}
113822024-01-24T19:10:21.305ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113832024-01-24T19:10:21.305ZLoading write count information from file "/tmp/test_repair_verify.data"
113842024-01-24T19:10:21.305ZRun Repair workload
113852024-01-24T19:10:21.305Z01/30 Write block 489 len 8192 data: 6 7
113862024-01-24T19:10:21.305Z02/30 Write block 161 len 28672 data: 12 12 13 10 11 9 10
113872024-01-24T19:10:21.305Z03/30 Write block 233 len 12288 data: 9 10 7
113882024-01-24T19:10:21.305Z04/30 Write block 299 len 36864 data: 6 6 6 6 5 4 5 5 6
113892024-01-24T19:10:21.305Z05/30 Read block 260 len 12288
113902024-01-24T19:10:21.310Z06/30 Write block 327 len 40960 data: 8 9 10 8 8 11 9 8 9 9
113912024-01-24T19:10:21.310Z07/30 Read block 354 len 8192
113922024-01-24T19:10:21.310Z08/30 Write block 89 len 8192 data: 5 6
113932024-01-24T19:10:21.310Z09/30 Read block 355 len 24576
113942024-01-24T19:10:21.310Z10/30 Write block 249 len 16384 data: 13 11 8 5
113952024-01-24T19:10:21.310Z11/30 Write block 366 len 16384 data: 11 10 10 9
113962024-01-24T19:10:21.310Z12/30 Read block 356 len 16384
113972024-01-24T19:10:21.310Z13/30 Read block 407 len 12288
113982024-01-24T19:10:21.311Z14/30 Read block 194 len 8192
113992024-01-24T19:10:21.317Z15/30 Read block 47 len 24576
114002024-01-24T19:10:21.318Z16/30 Read block 514 len 32768
114012024-01-24T19:10:21.318Z17/30 Read block 1 len 16384
114022024-01-24T19:10:21.319Z18/30 Read block 394 len 24576
114032024-01-24T19:10:21.324Z19/30 Write block 498 len 32768 data: 12 11 8 9 8 5 7 8
114042024-01-24T19:10:21.324Z20/30 Write block 586 len 32768 data: 8 8 8 7 7 9 12 11
114052024-01-24T19:10:21.324Z21/30 Flush
114062024-01-24T19:10:21.328Z22/30 Write block 395 len 12288 data: 10 11 8
114072024-01-24T19:10:21.328Z23/30 Read block 292 len 12288
114082024-01-24T19:10:21.331Z24/30 Write block 312 len 16384 data: 6 9 10 11
114092024-01-24T19:10:21.332Z25/30 Read block 357 len 16384
114102024-01-24T19:10:21.332Z26/30 Read block 226 len 24576
114112024-01-24T19:10:21.335Z27/30 Read block 24 len 20480
114122024-01-24T19:10:21.335Z28/30 Write block 470 len 36864 data: 6 8 12 9 10 11 10 9 9
114132024-01-24T19:10:21.338Z29/30 Write block 161 len 24576 data: 13 13 14 11 12 10
114142024-01-24T19:10:21.338Z30/30 Write block 546 len 36864 data: 10 11 9 8 8 9 7 6 6
114152024-01-24T19:10:21.338Z----------------------------------------------------------------
114162024-01-24T19:10:21.338Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114172024-01-24T19:10:21.339ZStates: Active Active Active
114182024-01-24T19:10:21.339ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114192024-01-24T19:10:21.339Z 1 Acked 1000 Write 2 Done Done Done false
114202024-01-24T19:10:21.339Z 2 Acked 1001 Write 7 Sent Done Done false
114212024-01-24T19:10:21.339Z 3 Acked 1002 Write 3 Sent Done Done false
114222024-01-24T19:10:21.339Z 4 Acked 1003 Write 9 Sent Done Done false
114232024-01-24T19:10:21.339Z 5 Acked 1004 Read 3 Sent Done Done false
114242024-01-24T19:10:21.339Z 6 Acked 1005 Write 10 Sent Done Done false
114252024-01-24T19:10:21.339Z 7 Acked 1006 Read 2 Sent Done Done false
114262024-01-24T19:10:21.339Z 8 Acked 1007 Write 2 Sent Done Done false
114272024-01-24T19:10:21.343Z 9 Acked 1008 Read 6 Sent Done Done false
114282024-01-24T19:10:21.343Z 10 Acked 1009 Write 4 Sent Done Done false
114292024-01-24T19:10:21.343Z 11 Acked 1010 Write 4 Sent Done Done false
114302024-01-24T19:10:21.343Z 12 Acked 1011 Read 4 Sent Done Done false
114312024-01-24T19:10:21.343Z 13 Acked 1012 Read 3 Sent Done Done false
114322024-01-24T19:10:21.344Z 14 Acked 1013 Read 2 Sent Done Done false
114332024-01-24T19:10:21.344Z 15 Acked 1014 Read 6 Sent Done Done false
114342024-01-24T19:10:21.344Z 16 Acked 1015 Read 8 Sent Done Done false
114352024-01-24T19:10:21.344Z 17 Acked 1016 Read 4 Sent Done Done false
114362024-01-24T19:10:21.344Z 18 Acked 1017 Read 6 Sent Done Done false
114372024-01-24T19:10:21.344Z 19 Acked 1018 Write 8 Sent Done Done false
114382024-01-24T19:10:21.344Z 20 Acked 1019 Write 8 Sent Done Done false
114392024-01-24T19:10:21.344Z 21 Acked 1020 Flush 0 Sent Done Done false
114402024-01-24T19:10:21.344Z 22 Acked 1021 Write 3 Sent Done Done false
114412024-01-24T19:10:21.344Z 23 Acked 1022 Read 3 Sent Done Done false
114422024-01-24T19:10:21.345Z 24 Acked 1023 Write 4 Sent Done Done false
114432024-01-24T19:10:21.345Z 25 Acked 1024 Read 4 Sent Done Done false
114442024-01-24T19:10:21.345Z 26 Acked 1025 Read 6 Sent Done Done false
114452024-01-24T19:10:21.345Z 27 Acked 1026 Read 5 Sent Done Done false
114462024-01-24T19:10:21.345Z 28 Acked 1027 Write 9 Sent Done Done false
114472024-01-24T19:10:21.345Z 29 Acked 1028 Write 6 Sent Done Done false
114482024-01-24T19:10:21.345Z 30 Acked 1029 Write 9 Sent Sent Sent false
114492024-01-24T19:10:21.345Z STATES DS:0 DS:1 DS:2 TOTAL
114502024-01-24T19:10:21.345Z New 0 0 0 0
114512024-01-24T19:10:21.345Z Sent 29 1 1 31
114522024-01-24T19:10:21.345Z Done 1 29 29 59
114532024-01-24T19:10:21.345Z Skipped 0 0 0 0
114542024-01-24T19:10:21.345Z Error 0 0 0 0
114552024-01-24T19:10:21.345ZLast Flush: 0 1020 1020
114562024-01-24T19:10:21.345ZDownstairs last five completed:
114572024-01-24T19:10:21.345ZUpstairs last five completed: 30 29 28 27 26
114582024-01-24T19:10:21.345ZWrote out file "/tmp/test_repair_verify.data"
114592024-01-24T19:10:21.349Z
114602024-01-24T19:10:21.352ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114612024-01-24T19:10:21.378ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114622024-01-24T19:10:21.379Z 0 000-019 99 99 99 171 171 171 F F F
114632024-01-24T19:10:21.382Z 1 020-039 96 96 96 166 166 166 F F F
114642024-01-24T19:10:21.382Z 2 040-059 104 104 104 179 179 179 F F F
114652024-01-24T19:10:21.382Z 3 060-079 104 104 104 179 179 179 F F F
114662024-01-24T19:10:21.382Z 4 080-099 103 106 106 178 180 180 F F F <---
114672024-01-24T19:10:21.382Z 5 100-119 103 103 103 178 178 178 F F F
114682024-01-24T19:10:21.382Z 6 120-139 103 103 103 178 178 178 F F F
114692024-01-24T19:10:21.382Z 7 140-159 102 102 102 177 177 177 F F F
114702024-01-24T19:10:21.383Z 8 160-179 104 106 106 179 180 180 F T T <---
114712024-01-24T19:10:21.383Z 9 180-199 104 104 104 179 179 179 F F F
114722024-01-24T19:10:21.383Z 10 200-219 102 102 102 177 177 177 F F F
114732024-01-24T19:10:21.383Z 11 220-239 92 106 106 157 180 180 F F F <---
114742024-01-24T19:10:21.383Z 12 240-259 104 106 106 179 180 180 F F F <---
114752024-01-24T19:10:21.383Z 13 260-279 102 102 102 177 177 177 F F F
114762024-01-24T19:10:21.384Z 14 280-299 102 106 106 177 180 180 F F F <---
114772024-01-24T19:10:21.384Z 15 300-319 102 106 106 177 180 180 F T T <---
114782024-01-24T19:10:21.384Z 16 320-339 101 106 106 175 180 180 F F F <---
114792024-01-24T19:10:21.384Z 17 340-359 104 104 104 179 179 179 F F F
114802024-01-24T19:10:21.384Z 18 360-379 104 106 106 179 180 180 F F F <---
114812024-01-24T19:10:21.384Z 19 380-399 104 104 104 179 179 179 F T T <---
114822024-01-24T19:10:21.384Z 20 400-419 104 104 104 179 179 179 F F F
114832024-01-24T19:10:21.384Z 21 420-439 100 100 100 173 173 173 F F F
114842024-01-24T19:10:21.384Z 22 440-459 92 92 92 155 155 155 F F F
114852024-01-24T19:10:21.384Z 23 460-479 102 102 102 177 177 177 F T T <---
114862024-01-24T19:10:21.384Z 24 480-499 102 106 106 177 180 180 T F F <---
114872024-01-24T19:10:21.384Z 25 500-519 104 106 106 179 180 180 F F F <---
114882024-01-24T19:10:21.384Z 26 520-539 104 104 104 179 179 179 F F F
114892024-01-24T19:10:21.384Z 27 540-559 104 104 104 179 179 179 F T T <---
114902024-01-24T19:10:21.384Z 28 560-579 102 102 102 177 177 177 F F F
114912024-01-24T19:10:21.384Z 29 580-599 101 106 106 175 180 180 F F F <---
114922024-01-24T19:10:21.384ZMax gen: 106, Max flush: 180
114932024-01-24T19:10:21.384ZError: Difference in extent metadata found!
114942024-01-24T19:10:21.388ZA Difference in extent metadata is expected here
114952024-01-24T19:10:21.388ZOn loop 53
114962024-01-24T19:10:21.388Z
114972024-01-24T19:10:21.388ZVerifying data now
114982024-01-24T19:10:26.643ZNo change after verify
114992024-01-24T19:10:26.643ZLoop: 53 Downstairs dump after verify (and repair):
115002024-01-24T19:10:26.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115012024-01-24T19:10:26.648Z 0 000-019 99 99 99 171 171 171 F F F
115022024-01-24T19:10:26.648Z 1 020-039 96 96 96 166 166 166 F F F
115032024-01-24T19:10:26.648Z 2 040-059 104 104 104 179 179 179 F F F
115042024-01-24T19:10:26.649Z 3 060-079 104 104 104 179 179 179 F F F
115052024-01-24T19:10:26.649Z 4 080-099 106 106 106 180 180 180 F F F
115062024-01-24T19:10:26.649Z 5 100-119 103 103 103 178 178 178 F F F
115072024-01-24T19:10:26.649Z 6 120-139 103 103 103 178 178 178 F F F
115082024-01-24T19:10:26.649Z 7 140-159 102 102 102 177 177 177 F F F
115092024-01-24T19:10:26.649Z 8 160-179 107 107 107 181 181 181 F F F
115102024-01-24T19:10:26.649Z 9 180-199 104 104 104 179 179 179 F F F
115112024-01-24T19:10:26.649Z 10 200-219 102 102 102 177 177 177 F F F
115122024-01-24T19:10:26.649Z 11 220-239 106 106 106 180 180 180 F F F
115132024-01-24T19:10:26.649Z 12 240-259 106 106 106 180 180 180 F F F
115142024-01-24T19:10:26.649Z 13 260-279 102 102 102 177 177 177 F F F
115152024-01-24T19:10:26.649Z 14 280-299 106 106 106 180 180 180 F F F
115162024-01-24T19:10:26.649Z 15 300-319 107 107 107 181 181 181 F F F
115172024-01-24T19:10:26.649Z 16 320-339 106 106 106 180 180 180 F F F
115182024-01-24T19:10:26.649Z 17 340-359 104 104 104 179 179 179 F F F
115192024-01-24T19:10:26.649Z 18 360-379 106 106 106 180 180 180 F F F
115202024-01-24T19:10:26.649Z 19 380-399 107 107 107 181 181 181 F F F
115212024-01-24T19:10:26.649Z 20 400-419 104 104 104 179 179 179 F F F
115222024-01-24T19:10:26.649Z 21 420-439 100 100 100 173 173 173 F F F
115232024-01-24T19:10:26.649Z 22 440-459 92 92 92 155 155 155 F F F
115242024-01-24T19:10:26.650Z 23 460-479 107 107 107 181 181 181 F F F
115252024-01-24T19:10:26.650Z 24 480-499 106 106 106 180 180 180 F F F
115262024-01-24T19:10:26.650Z 25 500-519 106 106 106 180 180 180 F F F
115272024-01-24T19:10:26.650Z 26 520-539 104 104 104 179 179 179 F F F
115282024-01-24T19:10:26.650Z 27 540-559 107 107 107 181 181 181 F F F
115292024-01-24T19:10:26.650Z 28 560-579 102 102 102 177 177 177 F F F
115302024-01-24T19:10:26.650Z 29 580-599 106 106 106 180 180 180 F F F
115312024-01-24T19:10:26.650ZMax gen: 107, Max flush: 181
115322024-01-24T19:10:26.650Z
115332024-01-24T19:10:26.650ZBegin loop 54
115342024-01-24T19:10:26.650ZDownstairs to restart: 0
115352024-01-24T19:10:26.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.341460531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561}
115362024-01-24T19:10:26.775Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.341943968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561}
115372024-01-24T19:10:26.784Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.341992003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561}
115382024-01-24T19:10:26.784Z{"msg":"Crucible 1bbb104e-9211-4126-97d4-a164cd62f73e has session id: 9b4e02ab-0658-48b9-809e-7357dee8af43","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342082967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115392024-01-24T19:10:26.784Z{"msg":"Upstairs opts: Upstairs UUID: 1bbb104e-9211-4126-97d4-a164cd62f73e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342213658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115402024-01-24T19:10:26.785Z{"msg":"Crucible stats registered with UUID: 1bbb104e-9211-4126-97d4-a164cd62f73e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342259839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115412024-01-24T19:10:26.785ZCrucible runtime is spawned
115422024-01-24T19:10:26.785Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342590046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561}
115432024-01-24T19:10:26.785Z{"msg":"1bbb104e-9211-4126-97d4-a164cd62f73e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342649421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115442024-01-24T19:10:26.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342690171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115452024-01-24T19:10:26.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342751634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115462024-01-24T19:10:26.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342873578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115472024-01-24T19:10:26.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342918537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115482024-01-24T19:10:26.785Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.342957124Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115492024-01-24T19:10:26.785Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343046451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115502024-01-24T19:10:26.785Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343076891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115512024-01-24T19:10:26.785Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.34311695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115522024-01-24T19:10:26.785Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343213231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115532024-01-24T19:10:26.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343290995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115542024-01-24T19:10:26.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343547864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115552024-01-24T19:10:26.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.343595455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115562024-01-24T19:10:26.785Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.344553892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115572024-01-24T19:10:26.786Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.344639353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115582024-01-24T19:10:26.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.34471533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115592024-01-24T19:10:26.786Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.344760215Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115602024-01-24T19:10:26.786Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.345385638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115612024-01-24T19:10:26.786Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.3454712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115622024-01-24T19:10:26.786Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.345616803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115632024-01-24T19:10:26.786Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.345749367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115642024-01-24T19:10:26.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346399359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115652024-01-24T19:10:26.786Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346500464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115662024-01-24T19:10:26.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346531999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115672024-01-24T19:10:26.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346591171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115682024-01-24T19:10:26.786Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346627929Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115692024-01-24T19:10:26.786Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346719193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115702024-01-24T19:10:26.786Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.346816615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115712024-01-24T19:10:26.786Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347448221Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115722024-01-24T19:10:26.786Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347516173Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115732024-01-24T19:10:26.786Z{"msg":"[0]R flush_numbers[0..12]: [171, 166, 179, 179, 180, 178, 178, 177, 181, 179, 177, 180]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347561965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115742024-01-24T19:10:26.786Z{"msg":"[0]R generation[0..12]: [99, 96, 104, 104, 106, 103, 103, 102, 107, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347615017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115752024-01-24T19:10:26.786Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347643223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115762024-01-24T19:10:26.786Z{"msg":"[1]R flush_numbers[0..12]: [171, 166, 179, 179, 180, 178, 178, 177, 181, 179, 177, 180]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347687813Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115772024-01-24T19:10:26.786Z{"msg":"[1]R generation[0..12]: [99, 96, 104, 104, 106, 103, 103, 102, 107, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347714468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115782024-01-24T19:10:26.786Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
115792024-01-24T19:10:26.787Z,"time":"2024-01-24T19:10:25.34775877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115802024-01-24T19:10:26.787Z{"msg":"[2]R flush_numbers[0..12]: [171, 166, 179, 179, 180, 178, 178, 177, 181, 179, 177, 180]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347788485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115812024-01-24T19:10:26.787Z{"msg":"[2]R generation[0..12]: [99, 96, 104, 104, 106, 103, 103, 102, 107, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347832233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115822024-01-24T19:10:26.787Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347858984Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115832024-01-24T19:10:26.787Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347901746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115842024-01-24T19:10:26.787Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347927388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115852024-01-24T19:10:26.787Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.347948897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115862024-01-24T19:10:26.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348003709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115872024-01-24T19:10:26.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348031424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115882024-01-24T19:10:26.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.34807365Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115892024-01-24T19:10:26.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348099255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115902024-01-24T19:10:26.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348153188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115912024-01-24T19:10:26.788Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348178894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115922024-01-24T19:10:26.788Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348201499Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115932024-01-24T19:10:26.788Z{"msg":"1bbb104e-9211-4126-97d4-a164cd62f73e is now active with session: 9b4e02ab-0658-48b9-809e-7357dee8af43","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348246789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115942024-01-24T19:10:26.789Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348270272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561,"session_id":"9b4e02ab-0658-48b9-809e-7357dee8af43"}
115952024-01-24T19:10:26.789Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:25.348290819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1561}
115962024-01-24T19:10:26.789ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115972024-01-24T19:10:26.789ZLoading write count information from file "/tmp/test_repair_verify.data"
115982024-01-24T19:10:26.789ZRun Repair workload
115992024-01-24T19:10:26.789Z01/30 Write block 118 len 32768 data: 7 9 9 9 9 8 12 10
116002024-01-24T19:10:26.789Z02/30 Read block 551 len 8192
116012024-01-24T19:10:26.789Z03/30 Write block 47 len 4096 data: 8
116022024-01-24T19:10:26.790Z04/30 Read block 491 len 28672
116032024-01-24T19:10:26.790Z05/30 Read block 569 len 12288
116042024-01-24T19:10:26.790Z06/30 Write block 353 len 36864 data: 8 7 8 10 9 10 9 8 12
116052024-01-24T19:10:26.790Z07/30 Flush
116062024-01-24T19:10:26.790Z08/30 Write block 93 len 28672 data: 10 11 9 9 9 8 6
116072024-01-24T19:10:26.790Z09/30 Flush
116082024-01-24T19:10:26.790Z10/30 Write block 249 len 4096 data: 14
116092024-01-24T19:10:26.790Z11/30 Read block 590 len 16384
116102024-01-24T19:10:26.790Z12/30 Write block 131 len 24576 data: 10 10 7 9 9 11
116112024-01-24T19:10:26.790Z13/30 Write block 157 len 28672 data: 9 9 6 6 14 14 15
116122024-01-24T19:10:26.790Z14/30 Write block 138 len 16384 data: 10 11 9 7
116132024-01-24T19:10:26.790Z15/30 Write block 319 len 28672 data: 12 12 11 10 11 10 12
116142024-01-24T19:10:26.790Z16/30 Read block 290 len 28672
116152024-01-24T19:10:26.790Z17/30 Flush
116162024-01-24T19:10:26.790Z18/30 Write block 397 len 4096 data: 9
116172024-01-24T19:10:26.790Z19/30 Read block 15 len 16384
116182024-01-24T19:10:26.790Z20/30 Write block 251 len 40960 data: 9 6 5 5 7 8 6 7 8 7
116192024-01-24T19:10:26.790Z21/30 Read block 587 len 24576
116202024-01-24T19:10:26.790Z22/30 Write block 504 len 4096 data: 8
116212024-01-24T19:10:26.791Z23/30 Read block 436 len 16384
116222024-01-24T19:10:26.791Z24/30 Write block 21 len 40960 data: 8 8 7 9 9 10 11 12 12 12
116232024-01-24T19:10:26.791Z25/30 Write block 489 len 24576 data: 7 8 7 9 9 9
116242024-01-24T19:10:26.791Z26/30 Read block 319 len 32768
116252024-01-24T19:10:26.791Z27/30 Read block 535 len 16384
116262024-01-24T19:10:26.791Z28/30 Write block 49 len 20480 data: 11 13 10 9 9
116272024-01-24T19:10:26.791Z29/30 Write block 178 len 4096 data: 8
116282024-01-24T19:10:26.791Z30/30 Write block 84 len 12288 data: 13 10 11
116292024-01-24T19:10:26.791Z----------------------------------------------------------------
116302024-01-24T19:10:26.791Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
116312024-01-24T19:10:26.791ZStates: Active Active Active
116322024-01-24T19:10:26.791ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116332024-01-24T19:10:26.791Z 1 Acked 1000 Write 8 Done Done Done false
116342024-01-24T19:10:26.791Z 2 Acked 1001 Read 2 Done Done Done false
116352024-01-24T19:10:26.791Z 3 Acked 1002 Write 1 Sent Done Done false
116362024-01-24T19:10:26.791Z 4 Acked 1003 Read 7 Sent Done Done false
116372024-01-24T19:10:26.797Z 5 Acked 1004 Read 3 Sent Done Done false
116382024-01-24T19:10:26.797Z 6 Acked 1005 Write 9 Sent Done Done false
116392024-01-24T19:10:26.797Z 7 Acked 1006 Flush 0 Sent Done Done false
116402024-01-24T19:10:26.797Z 8 Acked 1007 Write 7 Sent Done Done false
116412024-01-24T19:10:26.797Z 9 Acked 1008 Flush 0 Sent Done Done false
116422024-01-24T19:10:26.797Z 10 Acked 1009 Write 1 Sent Done Done false
116432024-01-24T19:10:26.797Z 11 Acked 1010 Read 4 Sent Done Done false
116442024-01-24T19:10:26.797Z 12 Acked 1011 Write 6 Sent Done Done false
116452024-01-24T19:10:26.797Z 13 Acked 1012 Write 7 Sent Done Done false
116462024-01-24T19:10:26.797Z 14 Acked 1013 Write 4 Sent Done Done false
116472024-01-24T19:10:26.797Z 15 Acked 1014 Write 7 Sent Done Done false
116482024-01-24T19:10:26.797Z 16 Acked 1015 Read 7 Sent Done Done false
116492024-01-24T19:10:26.797Z 17 Acked 1016 Flush 0 Sent Done Done false
116502024-01-24T19:10:26.797Z 18 Acked 1017 Write 1 Sent Done Done false
116512024-01-24T19:10:26.797Z 19 Acked 1018 Read 4 Sent Done Done false
116522024-01-24T19:10:26.798Z 20 Acked 1019 Write 10 Sent Done Done false
116532024-01-24T19:10:26.798Z 21 Acked 1020 Read 6 Sent Done Done false
116542024-01-24T19:10:26.798Z 22 Acked 1021 Write 1 Sent Done Done false
116552024-01-24T19:10:26.798Z 23 Acked 1022 Read 4 Sent Done Done false
116562024-01-24T19:10:26.798Z 24 Acked 1023 Write 10 Sent Done Done false
116572024-01-24T19:10:26.798Z 25 Acked 1024 Write 6 Sent Done Done false
116582024-01-24T19:10:26.798Z 26 Acked 1025 Read 8 Sent Done Done false
116592024-01-24T19:10:26.799Z 27 Acked 1026 Read 4 Sent Done Done false
116602024-01-24T19:10:26.799Z 28 Acked 1027 Write 5 Sent Done Done false
116612024-01-24T19:10:26.799Z 29 Acked 1028 Write 1 Sent Done Done false
116622024-01-24T19:10:26.799Z 30 Acked 1029 Write 3 Sent Sent Sent false
116632024-01-24T19:10:26.799Z STATES DS:0 DS:1 DS:2 TOTAL
116642024-01-24T19:10:26.799Z New 0 0 0 0
116652024-01-24T19:10:26.799Z Sent 28 1 1 30
116662024-01-24T19:10:26.799Z Done 2 29 29 60
116672024-01-24T19:10:26.799Z Skipped 0 0 0 0
116682024-01-24T19:10:26.799Z Error 0 0 0 0
116692024-01-24T19:10:26.799ZLast Flush: 0 1016 1016
116702024-01-24T19:10:26.799ZDownstairs last five completed:
116712024-01-24T19:10:26.799ZUpstairs last five completed: 30 29 28 27 26
116722024-01-24T19:10:26.799ZWrote out file "/tmp/test_repair_verify.data"
116732024-01-24T19:10:26.799Z
116742024-01-24T19:10:26.799ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116752024-01-24T19:10:26.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116762024-01-24T19:10:26.799Z 0 000-019 99 99 99 171 171 171 F F F
116772024-01-24T19:10:26.799Z 1 020-039 96 96 96 166 166 166 F T T <---
116782024-01-24T19:10:26.799Z 2 040-059 104 108 108 179 182 182 F T T <---
116792024-01-24T19:10:26.799Z 3 060-079 104 104 104 179 179 179 F F F
116802024-01-24T19:10:26.799Z 4 080-099 106 108 108 180 183 183 F T T <---
116812024-01-24T19:10:26.799Z 5 100-119 103 108 108 178 182 182 T F F <---
116822024-01-24T19:10:26.800Z 6 120-139 103 108 108 178 184 184 T F F <---
116832024-01-24T19:10:26.800Z 7 140-159 102 108 108 177 184 184 F F F <---
116842024-01-24T19:10:26.800Z 8 160-179 107 108 108 181 184 184 F T T <---
116852024-01-24T19:10:26.800Z 9 180-199 104 104 104 179 179 179 F F F
116862024-01-24T19:10:26.800Z 10 200-219 102 102 102 177 177 177 F F F
116872024-01-24T19:10:26.800Z 11 220-239 106 106 106 180 180 180 F F F
116882024-01-24T19:10:26.800Z 12 240-259 106 108 108 180 184 184 F T T <---
116892024-01-24T19:10:26.800Z 13 260-279 102 102 102 177 177 177 F T T <---
116902024-01-24T19:10:26.800Z 14 280-299 106 106 106 180 180 180 F F F
116912024-01-24T19:10:26.800Z 15 300-319 107 108 108 181 184 184 F F F <---
116922024-01-24T19:10:26.800Z 16 320-339 106 108 108 180 184 184 F F F <---
116932024-01-24T19:10:26.800Z 17 340-359 104 108 108 179 182 182 F F F <---
116942024-01-24T19:10:26.800Z 18 360-379 106 108 108 180 182 182 F F F <---
116952024-01-24T19:10:26.800Z 19 380-399 107 107 107 181 181 181 F T T <---
116962024-01-24T19:10:26.800Z 20 400-419 104 104 104 179 179 179 F F F
116972024-01-24T19:10:26.800Z 21 420-439 100 100 100 173 173 173 F F F
116982024-01-24T19:10:26.800Z 22 440-459 92 92 92 155 155 155 F F F
116992024-01-24T19:10:26.800Z 23 460-479 107 107 107 181 181 181 F F F
117002024-01-24T19:10:26.800Z 24 480-499 106 106 106 180 180 180 F T T <---
117012024-01-24T19:10:26.800Z 25 500-519 106 106 106 180 180 180 F T T <---
117022024-01-24T19:10:26.800Z 26 520-539 104 104 104 179 179 179 F F F
117032024-01-24T19:10:26.801Z 27 540-559 107 107 107 181 181 181 F F F
117042024-01-24T19:10:26.801Z 28 560-579 102 102 102 177 177 177 F F F
117052024-01-24T19:10:26.801Z 29 580-599 106 106 106 180 180 180 F F F
117062024-01-24T19:10:26.801ZMax gen: 108, Max flush: 184
117072024-01-24T19:10:26.801ZError: Difference in extent metadata found!
117082024-01-24T19:10:26.801ZA Difference in extent metadata is expected here
117092024-01-24T19:10:26.801ZOn loop 54
117102024-01-24T19:10:26.801Z
117112024-01-24T19:10:26.801ZVerifying data now
117122024-01-24T19:10:32.241ZNo change after verify
117132024-01-24T19:10:32.241ZLoop: 54 Downstairs dump after verify (and repair):
117142024-01-24T19:10:32.263ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117152024-01-24T19:10:32.263Z 0 000-019 99 99 99 171 171 171 F F F
117162024-01-24T19:10:32.265Z 1 020-039 109 109 109 185 185 185 F F F
117172024-01-24T19:10:32.265Z 2 040-059 109 109 109 185 185 185 F F F
117182024-01-24T19:10:32.265Z 3 060-079 104 104 104 179 179 179 F F F
117192024-01-24T19:10:32.265Z 4 080-099 109 109 109 185 185 185 F F F
117202024-01-24T19:10:32.265Z 5 100-119 108 108 108 182 182 182 F F F
117212024-01-24T19:10:32.266Z 6 120-139 108 108 108 184 184 184 F F F
117222024-01-24T19:10:32.266Z 7 140-159 108 108 108 184 184 184 F F F
117232024-01-24T19:10:32.266Z 8 160-179 109 109 109 185 185 185 F F F
117242024-01-24T19:10:32.266Z 9 180-199 104 104 104 179 179 179 F F F
117252024-01-24T19:10:32.266Z 10 200-219 102 102 102 177 177 177 F F F
117262024-01-24T19:10:32.266Z 11 220-239 106 106 106 180 180 180 F F F
117272024-01-24T19:10:32.267Z 12 240-259 109 109 109 185 185 185 F F F
117282024-01-24T19:10:32.267Z 13 260-279 109 109 109 185 185 185 F F F
117292024-01-24T19:10:32.267Z 14 280-299 106 106 106 180 180 180 F F F
117302024-01-24T19:10:32.267Z 15 300-319 108 108 108 184 184 184 F F F
117312024-01-24T19:10:32.267Z 16 320-339 108 108 108 184 184 184 F F F
117322024-01-24T19:10:32.267Z 17 340-359 108 108 108 182 182 182 F F F
117332024-01-24T19:10:32.267Z 18 360-379 108 108 108 182 182 182 F F F
117342024-01-24T19:10:32.267Z 19 380-399 109 109 109 185 185 185 F F F
117352024-01-24T19:10:32.267Z 20 400-419 104 104 104 179 179 179 F F F
117362024-01-24T19:10:32.267Z 21 420-439 100 100 100 173 173 173 F F F
117372024-01-24T19:10:32.267Z 22 440-459 92 92 92 155 155 155 F F F
117382024-01-24T19:10:32.268Z 23 460-479 107 107 107 181 181 181 F F F
117392024-01-24T19:10:32.268Z 24 480-499 109 109 109 185 185 185 F F F
117402024-01-24T19:10:32.269Z 25 500-519 109 109 109 185 185 185 F F F
117412024-01-24T19:10:32.269Z 26 520-539 104 104 104 179 179 179 F F F
117422024-01-24T19:10:32.269Z 27 540-559 107 107 107 181 181 181 F F F
117432024-01-24T19:10:32.269Z 28 560-579 102 102 102 177 177 177 F F F
117442024-01-24T19:10:32.269Z 29 580-599 106 106 106 180 180 180 F F F
117452024-01-24T19:10:32.269ZMax gen: 109, Max flush: 185
117462024-01-24T19:10:32.269Z
117472024-01-24T19:10:32.269ZBegin loop 55
117482024-01-24T19:10:32.269ZDownstairs to restart: 0
117492024-01-24T19:10:32.269Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.88134644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569}
117502024-01-24T19:10:32.270Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.881820267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569}
117512024-01-24T19:10:32.270Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.881872218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569}
117522024-01-24T19:10:32.270Z{"msg":"Crucible 6fe57ff7-7161-4602-acf1-d430089bfe94 has session id: bdca6afb-5325-45e9-b9a7-3fa0f1730132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.881896559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117532024-01-24T19:10:32.270Z{"msg":"Upstairs opts: Upstairs UUID: 6fe57ff7-7161-4602-acf1-d430089bfe94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.881931404Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117542024-01-24T19:10:32.270Z{"msg":"Crucible stats registered with UUID: 6fe57ff7-7161-4602-acf1-d430089bfe94","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.881959347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117552024-01-24T19:10:32.271ZCrucible runtime is spawned
117562024-01-24T19:10:32.271Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882171547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569}
117572024-01-24T19:10:32.271Z{"msg":"6fe57ff7-7161-4602-acf1-d430089bfe94 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882529332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117582024-01-24T19:10:32.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882619158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117592024-01-24T19:10:32.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882664697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117602024-01-24T19:10:32.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882734732Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117612024-01-24T19:10:32.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882780699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117622024-01-24T19:10:32.272Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.882851996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117632024-01-24T19:10:32.272Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883050036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117642024-01-24T19:10:32.272Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883129919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117652024-01-24T19:10:32.272Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883178008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117662024-01-24T19:10:32.273Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883211971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117672024-01-24T19:10:32.273Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883291909Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117682024-01-24T19:10:32.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883338636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117692024-01-24T19:10:32.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.883401048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117702024-01-24T19:10:32.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.884290783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117712024-01-24T19:10:32.274Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.88433194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117722024-01-24T19:10:32.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.88442027Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117732024-01-24T19:10:32.275Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.8844899Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117742024-01-24T19:10:32.275Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.884603462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117752024-01-24T19:10:32.275Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.884635369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117762024-01-24T19:10:32.288Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.885474612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117772024-01-24T19:10:32.288Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.885565573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117782024-01-24T19:10:32.288Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.885700454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117792024-01-24T19:10:32.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.88635926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117802024-01-24T19:10:32.288Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886451029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117812024-01-24T19:10:32.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886542243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117822024-01-24T19:10:32.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.88660625Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117832024-01-24T19:10:32.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886649059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117842024-01-24T19:10:32.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886685445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117852024-01-24T19:10:32.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886749447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117862024-01-24T19:10:32.289Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886817527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117872024-01-24T19:10:32.289Z{"msg":"[0]R flush_numbers[0..12]: [171, 185, 185, 179, 185, 182, 184, 184, 185, 179, 177, 180]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886861107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117882024-01-24T19:10:32.289Z{"msg":"[0]R generation[0..12]: [99, 109, 109, 104, 109, 108, 108, 108, 109, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886922258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117892024-01-24T19:10:32.289Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.886968951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117902024-01-24T19:10:32.289Z{"msg":"[1]R flush_numbers[0..12]: [171, 185, 185, 179, 185, 182, 184, 184, 185, 179, 177, 180]",Wait for a query_work_queue command to finish before sending IO
117912024-01-24T19:10:32.289Z"v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887056747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117922024-01-24T19:10:32.289Z{"msg":"[1]R generation[0..12]: [99, 109, 109, 104, 109, 108, 108, 108, 109, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887120899Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117932024-01-24T19:10:32.289Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887165868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117942024-01-24T19:10:32.289Z{"msg":"[2]R flush_numbers[0..12]: [171, 185, 185, 179, 185, 182, 184, 184, 185, 179, 177, 180]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887230624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117952024-01-24T19:10:32.289Z{"msg":"[2]R generation[0..12]: [99, 109, 109, 104, 109, 108, 108, 108, 109, 104, 102, 106]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887275297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117962024-01-24T19:10:32.289Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887331837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117972024-01-24T19:10:32.290Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887375146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117982024-01-24T19:10:32.290Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887438032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
117992024-01-24T19:10:32.290Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887501494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118002024-01-24T19:10:32.290Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887544195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118012024-01-24T19:10:32.290Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887600304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118022024-01-24T19:10:32.290Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887638925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118032024-01-24T19:10:32.290Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887695727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118042024-01-24T19:10:32.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118052024-01-24T19:10:32.294Z30,"time":"2024-01-24T19:10:30.88774345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118062024-01-24T19:10:32.294Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887786699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118072024-01-24T19:10:32.294Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887846867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118082024-01-24T19:10:32.295Z{"msg":"6fe57ff7-7161-4602-acf1-d430089bfe94 is now active with session: bdca6afb-5325-45e9-b9a7-3fa0f1730132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887889387Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118092024-01-24T19:10:32.295Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887941495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569,"session_id":"bdca6afb-5325-45e9-b9a7-3fa0f1730132"}
118102024-01-24T19:10:32.295Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:30.887981627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1569}
118112024-01-24T19:10:32.295ZLoading write count information from file "/tmp/test_repair_verify.data"
118122024-01-24T19:10:32.295ZRun Repair workload
118132024-01-24T19:10:32.295Z01/30 Write block 206 len 12288 data: 6 7 8
118142024-01-24T19:10:32.339Z02/30 Read block 425 len 24576
118152024-01-24T19:10:32.339Z03/30 Flush
118162024-01-24T19:10:32.339Z04/30 Write block 135 len 16384 data: 10 12 12 11
118172024-01-24T19:10:32.339Z05/30 Read block 87 len 12288
118182024-01-24T19:10:32.339Z06/30 Read block 276 len 4096
118192024-01-24T19:10:32.339Z07/30 Write block 328 len 16384 data: 10 11 9 9
118202024-01-24T19:10:32.339Z08/30 Write block 390 len 4096 data: 6
118212024-01-24T19:10:32.339Z09/30 Write block 292 len 12288 data: 8 10 12
118222024-01-24T19:10:32.339Z10/30 Read block 333 len 8192
118232024-01-24T19:10:32.339Z11/30 Read block 212 len 20480
118242024-01-24T19:10:32.339Z12/30 Write block 152 len 28672 data: 11 10 10 8 8 10 10
118252024-01-24T19:10:32.339Z13/30 Write block 406 len 40960 data: 8 8 9 5 8 8 7 7 4 4
118262024-01-24T19:10:32.361Z14/30 Write block 180 len 4096 data: 8
118272024-01-24T19:10:32.361Z15/30 Write block 28 len 8192 data: 13 13
118282024-01-24T19:10:32.361Z16/30 Read block 476 len 28672
118292024-01-24T19:10:32.361Z17/30 Read block 309 len 24576
118302024-01-24T19:10:32.361Z18/30 Flush
118312024-01-24T19:10:32.362Z19/30 Write block 470 len 32768 data: 7 9 13 10 11 12 11 10
118322024-01-24T19:10:32.362Z20/30 Write block 221 len 20480 data: 8 9 6 7 9
118332024-01-24T19:10:32.362Z21/30 Write block 579 len 20480 data: 6 6 8 7 8
118342024-01-24T19:10:32.362Z22/30 Flush
118352024-01-24T19:10:32.362Z23/30 Write block 144 len 40960 data: 5 6 8 10 8 10 10 12 12 11
118362024-01-24T19:10:32.362Z24/30 Read block 329 len 32768
118372024-01-24T19:10:32.362Z25/30 Read block 399 len 8192
118382024-01-24T19:10:32.362Z26/30 Read block 321 len 20480
118392024-01-24T19:10:32.362Z27/30 Read block 410 len 4096
118402024-01-24T19:10:32.362Z28/30 Read block 550 len 4096
118412024-01-24T19:10:32.362Z29/30 Read block 300 len 4096
118422024-01-24T19:10:32.362Z30/30 Read block 449 len 20480
118432024-01-24T19:10:32.362Z----------------------------------------------------------------
118442024-01-24T19:10:32.362Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:27
118452024-01-24T19:10:32.362ZStates: Active Active Active
118462024-01-24T19:10:32.362ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118472024-01-24T19:10:32.362Z 4 Acked 1003 Write 4 Done Done Done false
118482024-01-24T19:10:32.362Z 5 Acked 1004 Read 3 Done Done Done false
118492024-01-24T19:10:32.362Z 6 Acked 1005 Read 1 Done Done Done false
118502024-01-24T19:10:32.362Z 7 Acked 1006 Write 4 Done Done Done false
118512024-01-24T19:10:32.362Z 8 Acked 1007 Write 1 Done Done Done false
118522024-01-24T19:10:32.362Z 9 Acked 1008 Write 3 Done Done Done false
118532024-01-24T19:10:32.363Z 10 Acked 1009 Read 2 Done Done Done false
118542024-01-24T19:10:32.363Z 11 Acked 1010 Read 5 Done Done Done false
118552024-01-24T19:10:32.363Z 12 Acked 1011 Write 7 Done Done Done false
118562024-01-24T19:10:32.363Z 13 Acked 1012 Write 10 Done Done Done false
118572024-01-24T19:10:32.363Z 14 Acked 1013 Write 1 Sent Done Done false
118582024-01-24T19:10:32.363Z 15 Acked 1014 Write 2 Sent Done Done false
118592024-01-24T19:10:32.363Z 16 Acked 1015 Read 7 Sent Done Done false
118602024-01-24T19:10:32.363Z 17 Acked 1016 Read 6 Sent Done Done false
118612024-01-24T19:10:32.363Z 18 Acked 1017 Flush 0 Sent Done Done false
118622024-01-24T19:10:32.363Z 19 Acked 1018 Write 8 Sent Done Done false
118632024-01-24T19:10:32.363Z 20 Acked 1019 Write 5 Sent Done Done false
118642024-01-24T19:10:32.363Z 21 Acked 1020 Write 5 Sent Done Done false
118652024-01-24T19:10:32.363Z 22 Acked 1021 Flush 0 Sent Done Done false
118662024-01-24T19:10:32.363Z 23 Acked 1022 Write 10 Sent Done Done false
118672024-01-24T19:10:32.363Z 24 Acked 1023 Read 8 Sent Done Done false
118682024-01-24T19:10:32.363Z 25 Acked 1024 Read 2 Sent Done Done false
118692024-01-24T19:10:32.363Z 26 Acked 1025 Read 5 Sent Done Done false
118702024-01-24T19:10:32.363Z 27 Acked 1026 Read 1 Sent Done Done false
118712024-01-24T19:10:32.363Z 28 Acked 1027 Read 1 Sent Done Done false
118722024-01-24T19:10:32.363Z 29 Acked 1028 Read 1 Sent Done Done false
118732024-01-24T19:10:32.363Z 30 Acked 1029 Read 5 Sent Sent Done false
118742024-01-24T19:10:32.364Z STATES DS:0 DS:1 DS:2 TOTAL
118752024-01-24T19:10:32.364Z New 0 0 0 0
118762024-01-24T19:10:32.364Z Sent 17 1 0 18
118772024-01-24T19:10:32.364Z Done 10 26 27 63
118782024-01-24T19:10:32.365Z Skipped 0 0 0 0
118792024-01-24T19:10:32.365Z Error 0 0 0 0
118802024-01-24T19:10:32.365ZLast Flush: 1002 1021 1021
118812024-01-24T19:10:32.365ZDownstairs last five completed: 1002 1001 1000
118822024-01-24T19:10:32.365ZUpstairs last five completed: 30 29 28 27 26
118832024-01-24T19:10:32.365ZWrote out file "/tmp/test_repair_verify.data"
118842024-01-24T19:10:32.365Z
118852024-01-24T19:10:32.365ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118862024-01-24T19:10:32.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118872024-01-24T19:10:32.365Z 0 000-019 99 99 99 171 171 171 F F F
118882024-01-24T19:10:32.365Z 1 020-039 109 110 110 185 187 187 F F F <---
118892024-01-24T19:10:32.365Z 2 040-059 109 109 109 185 185 185 F F F
118902024-01-24T19:10:32.365Z 3 060-079 104 104 104 179 179 179 F F F
118912024-01-24T19:10:32.365Z 4 080-099 109 109 109 185 185 185 F F F
118922024-01-24T19:10:32.365Z 5 100-119 108 108 108 182 182 182 F F F
118932024-01-24T19:10:32.365Z 6 120-139 108 110 110 184 187 187 T F F <---
118942024-01-24T19:10:32.365Z 7 140-159 108 110 110 184 187 187 T T T <---
118952024-01-24T19:10:32.365Z 8 160-179 109 109 109 185 185 185 F F F
118962024-01-24T19:10:32.365Z 9 180-199 104 110 110 179 187 187 F F F <---
118972024-01-24T19:10:32.365Z 10 200-219 110 110 110 186 186 186 F F F
118982024-01-24T19:10:32.365Z 11 220-239 106 110 110 180 188 188 F F F <---
118992024-01-24T19:10:32.368Z 12 240-259 109 109 109 185 185 185 F F F
119002024-01-24T19:10:32.368Z 13 260-279 109 109 109 185 185 185 F F F
119012024-01-24T19:10:32.368Z 14 280-299 106 110 110 180 187 187 T F F <---
119022024-01-24T19:10:32.368Z 15 300-319 108 108 108 184 184 184 F F F
119032024-01-24T19:10:32.368Z 16 320-339 108 110 110 184 187 187 T F F <---
119042024-01-24T19:10:32.368Z 17 340-359 108 108 108 182 182 182 F F F
119052024-01-24T19:10:32.370Z 18 360-379 108 108 108 182 182 182 F F F
119062024-01-24T19:10:32.370Z 19 380-399 109 110 110 185 187 187 T F F <---
119072024-01-24T19:10:32.370Z 20 400-419 104 110 110 179 187 187 T F F <---
119082024-01-24T19:10:32.370Z 21 420-439 100 100 100 173 173 173 F F F
119092024-01-24T19:10:32.370Z 22 440-459 92 92 92 155 155 155 F F F
119102024-01-24T19:10:32.370Z 23 460-479 107 110 110 181 188 188 F F F <---
119112024-01-24T19:10:32.370Z 24 480-499 109 109 109 185 185 185 F F F
119122024-01-24T19:10:32.370Z 25 500-519 109 109 109 185 185 185 F F F
119132024-01-24T19:10:32.370Z 26 520-539 104 104 104 179 179 179 F F F
119142024-01-24T19:10:32.378Z 27 540-559 107 107 107 181 181 181 F F F
119152024-01-24T19:10:32.378Z 28 560-579 102 110 110 177 188 188 F F F <---
119162024-01-24T19:10:32.378Z 29 580-599 106 110 110 180 188 188 F F F <---
119172024-01-24T19:10:32.378ZMax gen: 110, Max flush: 188
119182024-01-24T19:10:32.378ZError: Difference in extent metadata found!
119192024-01-24T19:10:32.379ZA Difference in extent metadata is expected here
119202024-01-24T19:10:32.379ZOn loop 55
119212024-01-24T19:10:32.379Z
119222024-01-24T19:10:32.379ZVerifying data now
119232024-01-24T19:10:38.377ZNo change after verify
119242024-01-24T19:10:38.377ZLoop: 55 Downstairs dump after verify (and repair):
119252024-01-24T19:10:38.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119262024-01-24T19:10:38.401Z 0 000-019 99 99 99 171 171 171 F F F
119272024-01-24T19:10:38.412Z 1 020-039 110 110 110 187 187 187 F F F
119282024-01-24T19:10:38.412Z 2 040-059 109 109 109 185 185 185 F F F
119292024-01-24T19:10:38.412Z 3 060-079 104 104 104 179 179 179 F F F
119302024-01-24T19:10:38.412Z 4 080-099 109 109 109 185 185 185 F F F
119312024-01-24T19:10:38.413Z 5 100-119 108 108 108 182 182 182 F F F
119322024-01-24T19:10:38.413Z 6 120-139 110 110 110 187 187 187 F F F
119332024-01-24T19:10:38.413Z 7 140-159 111 111 111 189 189 189 F F F
119342024-01-24T19:10:38.413Z 8 160-179 109 109 109 185 185 185 F F F
119352024-01-24T19:10:38.413Z 9 180-199 110 110 110 187 187 187 F F F
119362024-01-24T19:10:38.414Z 10 200-219 110 110 110 186 186 186 F F F
119372024-01-24T19:10:38.414Z 11 220-239 110 110 110 188 188 188 F F F
119382024-01-24T19:10:38.415Z 12 240-259 109 109 109 185 185 185 F F F
119392024-01-24T19:10:38.415Z 13 260-279 109 109 109 185 185 185 F F F
119402024-01-24T19:10:38.415Z 14 280-299 110 110 110 187 187 187 F F F
119412024-01-24T19:10:38.415Z 15 300-319 108 108 108 184 184 184 F F F
119422024-01-24T19:10:38.415Z 16 320-339 110 110 110 187 187 187 F F F
119432024-01-24T19:10:38.415Z 17 340-359 108 108 108 182 182 182 F F F
119442024-01-24T19:10:38.416Z 18 360-379 108 108 108 182 182 182 F F F
119452024-01-24T19:10:38.416Z 19 380-399 110 110 110 187 187 187 F F F
119462024-01-24T19:10:38.416Z 20 400-419 110 110 110 187 187 187 F F F
119472024-01-24T19:10:38.416Z 21 420-439 100 100 100 173 173 173 F F F
119482024-01-24T19:10:38.416Z 22 440-459 92 92 92 155 155 155 F F F
119492024-01-24T19:10:38.416Z 23 460-479 110 110 110 188 188 188 F F F
119502024-01-24T19:10:38.417Z 24 480-499 109 109 109 185 185 185 F F F
119512024-01-24T19:10:38.417Z 25 500-519 109 109 109 185 185 185 F F F
119522024-01-24T19:10:38.417Z 26 520-539 104 104 104 179 179 179 F F F
119532024-01-24T19:10:38.417Z 27 540-559 107 107 107 181 181 181 F F F
119542024-01-24T19:10:38.417Z 28 560-579 110 110 110 188 188 188 F F F
119552024-01-24T19:10:38.417Z 29 580-599 110 110 110 188 188 188 F F F
119562024-01-24T19:10:38.417ZMax gen: 111, Max flush: 189
119572024-01-24T19:10:38.418Z
119582024-01-24T19:10:38.418ZBegin loop 56
119592024-01-24T19:10:38.418ZDownstairs to restart: 2
119602024-01-24T19:10:38.436Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.100394198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577}
119612024-01-24T19:10:38.448Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.100822525Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577}
119622024-01-24T19:10:38.448Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.100900105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577}
119632024-01-24T19:10:38.448Z{"msg":"Crucible 59be881b-02f1-4103-bb70-9eb7e07a6789 has session id: b0bd260d-5250-4ea3-b63f-0ccb19dd3380","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.100943508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119642024-01-24T19:10:38.448Z{"msg":"Upstairs opts: Upstairs UUID: 59be881b-02f1-4103-bb70-9eb7e07a6789, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
119652024-01-24T19:10:38.448Z,"time":"2024-01-24T19:10:37.101034006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119662024-01-24T19:10:38.449Z{"msg":"Crucible stats registered with UUID: 59be881b-02f1-4103-bb70-9eb7e07a6789","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101086233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119672024-01-24T19:10:38.449Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101156303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577}
119682024-01-24T19:10:38.449Z{"msg":"59be881b-02f1-4103-bb70-9eb7e07a6789 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10153094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119692024-01-24T19:10:38.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10158601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119702024-01-24T19:10:38.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101658145Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119712024-01-24T19:10:38.451Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101684398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119722024-01-24T19:10:38.451Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101711286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119732024-01-24T19:10:38.451Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101753409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119742024-01-24T19:10:38.451Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101795156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119752024-01-24T19:10:38.451Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101855894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119762024-01-24T19:10:38.451Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.101902726Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119772024-01-24T19:10:38.451Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10194841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119782024-01-24T19:10:38.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.102212034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119792024-01-24T19:10:38.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.102361704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119802024-01-24T19:10:38.451Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.102408044Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119812024-01-24T19:10:38.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.103631411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119822024-01-24T19:10:38.452Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.103708411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119832024-01-24T19:10:38.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.103888424Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119842024-01-24T19:10:38.452Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.103990214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119852024-01-24T19:10:38.452Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.104079094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119862024-01-24T19:10:38.452Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.104125332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119872024-01-24T19:10:38.452Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.104885551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119882024-01-24T19:10:38.452Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.104947869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119892024-01-24T19:10:38.452Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.105393196Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119902024-01-24T19:10:38.452Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.105936121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119912024-01-24T19:10:38.453Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106021846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119922024-01-24T19:10:38.453Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10605001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119932024-01-24T19:10:38.453Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106135172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119942024-01-24T19:10:38.453Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106182131Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119952024-01-24T19:10:38.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106207748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119962024-01-24T19:10:38.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106326805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119972024-01-24T19:10:38.454Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106381015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119982024-01-24T19:10:38.454Z{"msg":"[0]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 187, 189, 185, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106409322Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
119992024-01-24T19:10:38.454Z{"msg":"[0]R generation[0..12]: [99, 110, 109, 104, 109, 108, 110, 111, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106459964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120002024-01-24T19:10:38.454Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10648692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120012024-01-24T19:10:38.455Z{"msg":"[1]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 187, 189, 185, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106531892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120022024-01-24T19:10:38.455Z{"msg":"[1]R generation[0..12]: [99, 110, 109, 104, 109, 108, 110, 111, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106558471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120032024-01-24T19:10:38.455Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106602706Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120042024-01-24T19:10:38.455Z{"msg":"[2]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 187, 189, 185, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.10662968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120052024-01-24T19:10:38.455Z{"msg":"[2]R generation[0..12]: [99, 110, 109, 104, 109, 108, 110, 111, 109, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106653034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120062024-01-24T19:10:38.455Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106676698Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120072024-01-24T19:10:38.455ZWait for a query_work_queue command to finish before sending IO
120082024-01-24T19:10:38.455Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106724334Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120092024-01-24T19:10:38.456Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106750359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120102024-01-24T19:10:38.456Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106801169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120112024-01-24T19:10:38.456Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106826438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120122024-01-24T19:10:38.457Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106874973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120132024-01-24T19:10:38.457Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106898187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120142024-01-24T19:10:38.458Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106921532Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120152024-01-24T19:10:38.458Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.106988373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120162024-01-24T19:10:38.458Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.107013972Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120172024-01-24T19:10:38.458Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.107034616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120182024-01-24T19:10:38.458Z{"msg":"59be881b-02f1-4103-bb70-9eb7e07a6789 is now active with session: b0bd260d-5250-4ea3-b63f-0ccb19dd3380","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.107091017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120192024-01-24T19:10:38.458Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.107114128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577,"session_id":"b0bd260d-5250-4ea3-b63f-0ccb19dd3380"}
120202024-01-24T19:10:38.458Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:37.107134714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1577}
120212024-01-24T19:10:38.458ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120222024-01-24T19:10:38.460ZLoading write count information from file "/tmp/test_repair_verify.data"
120232024-01-24T19:10:38.460ZRun Repair workload
120242024-01-24T19:10:38.460Z01/30 Write block 254 len 24576 data: 6 8 9 7 8 9
120252024-01-24T19:10:38.460Z02/30 Read block 297 len 40960
120262024-01-24T19:10:38.465Z03/30 Flush
120272024-01-24T19:10:38.465Z04/30 Flush
120282024-01-24T19:10:38.465Z05/30 Write block 525 len 28672 data: 10 11 10 13 12 11 9
120292024-01-24T19:10:38.465Z06/30 Write block 361 len 12288 data: 13 11 10
120302024-01-24T19:10:38.465Z07/30 Read block 1 len 8192
120312024-01-24T19:10:38.465Z08/30 Write block 275 len 8192 data: 10 8
120322024-01-24T19:10:38.465Z09/30 Read block 308 len 28672
120332024-01-24T19:10:38.465Z10/30 Read block 157 len 36864
120342024-01-24T19:10:38.465Z11/30 Read block 515 len 32768
120352024-01-24T19:10:38.465Z12/30 Flush
120362024-01-24T19:10:38.465Z13/30 Flush
120372024-01-24T19:10:38.465Z14/30 Write block 364 len 32768 data: 12 12 12 11 11 10 10 9
120382024-01-24T19:10:38.466Z15/30 Write block 298 len 24576 data: 8 7 7 7 7 6
120392024-01-24T19:10:38.466Z16/30 Read block 411 len 28672
120402024-01-24T19:10:38.469Z17/30 Read block 363 len 16384
120412024-01-24T19:10:38.469Z18/30 Read block 140 len 8192
120422024-01-24T19:10:38.469Z19/30 Write block 596 len 4096 data: 7
120432024-01-24T19:10:38.469Z20/30 Read block 201 len 28672
120442024-01-24T19:10:38.476Z21/30 Read block 497 len 36864
120452024-01-24T19:10:38.479Z22/30 Write block 494 len 12288 data: 10 9 12
120462024-01-24T19:10:38.479Z23/30 Write block 236 len 36864 data: 6 5 4 5 7 8 10 13 12
120472024-01-24T19:10:38.479Z24/30 Flush
120482024-01-24T19:10:38.482Z25/30 Write block 230 len 40960 data: 8 8 8 10 11 8 7 6 5 6
120492024-01-24T19:10:38.482Z26/30 Read block 533 len 28672
120502024-01-24T19:10:38.499Z27/30 Read block 279 len 40960
120512024-01-24T19:10:38.499Z28/30 Write block 133 len 28672 data: 8 10 11 13 13 12 12
120522024-01-24T19:10:38.499Z29/30 Read block 557 len 40960
120532024-01-24T19:10:38.503Z30/30 Read block 349 len 40960
120542024-01-24T19:10:38.503Z----------------------------------------------------------------
120552024-01-24T19:10:38.503Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:26
120562024-01-24T19:10:38.503ZStates: Active Active Active
120572024-01-24T19:10:38.504ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
120582024-01-24T19:10:38.504Z 5 Acked 1004 Write 7 Done Done Done false
120592024-01-24T19:10:38.504Z 6 Acked 1005 Write 3 Done Done Sent false
120602024-01-24T19:10:38.504Z 7 Acked 1006 Read 2 Done Done Sent false
120612024-01-24T19:10:38.504Z 8 Acked 1007 Write 2 Done Done Sent false
120622024-01-24T19:10:38.504Z 9 Acked 1008 Read 7 Done Done Sent false
120632024-01-24T19:10:38.504Z 10 Acked 1009 Read 9 Done Done Sent false
120642024-01-24T19:10:38.504Z 11 Acked 1010 Read 8 Done Done Sent false
120652024-01-24T19:10:38.504Z 12 Acked 1011 Flush 0 Done Done Sent false
120662024-01-24T19:10:38.508Z 13 Acked 1012 Flush 0 Done Done Sent false
120672024-01-24T19:10:38.508Z 14 Acked 1013 Write 8 Done Done Sent false
120682024-01-24T19:10:38.508Z 15 Acked 1014 Write 6 Done Done Sent false
120692024-01-24T19:10:38.508Z 16 Acked 1015 Read 7 Done Done Sent false
120702024-01-24T19:10:38.509Z 17 Acked 1016 Read 4 Done Done Sent false
120712024-01-24T19:10:38.509Z 18 Acked 1017 Read 2 Done Done Sent false
120722024-01-24T19:10:38.509Z 19 Acked 1018 Write 1 Done Done Sent false
120732024-01-24T19:10:38.509Z 20 Acked 1019 Read 7 Done Done Sent false
120742024-01-24T19:10:38.509Z 21 Acked 1020 Read 9 Done Done Sent false
120752024-01-24T19:10:38.509Z 22 Acked 1021 Write 3 Done Done Sent false
120762024-01-24T19:10:38.509Z 23 Acked 1022 Write 9 Done Done Sent false
120772024-01-24T19:10:38.509Z 24 Acked 1023 Flush 0 Done Done Sent false
120782024-01-24T19:10:38.509Z 25 Acked 1024 Write 10 Done Done Sent false
120792024-01-24T19:10:38.509Z 26 Acked 1025 Read 7 Done Done Sent false
120802024-01-24T19:10:38.509Z 27 Acked 1026 Read 10 Done Done Sent false
120812024-01-24T19:10:38.510Z 28 Acked 1027 Write 7 Done Done Sent false
120822024-01-24T19:10:38.510Z 29 Acked 1028 Read 10 Done Done Sent false
120832024-01-24T19:10:38.510Z 30 Acked 1029 Read 10 Done Done Sent false
120842024-01-24T19:10:38.510Z STATES DS:0 DS:1 DS:2 TOTAL
120852024-01-24T19:10:38.510Z New 0 0 0 0
120862024-01-24T19:10:38.510Z Sent 0 0 25 25
120872024-01-24T19:10:38.510Z Done 26 26 1 53
120882024-01-24T19:10:38.510Z Skipped 0 0 0 0
120892024-01-24T19:10:38.510Z Error 0 0 0 0
120902024-01-24T19:10:38.510ZLast Flush: 1023 1023 1003
120912024-01-24T19:10:38.510ZDownstairs last five completed: 1003 1002 1001 1000
120922024-01-24T19:10:38.510ZUpstairs last five completed: 30 29 28 27 26
120932024-01-24T19:10:38.510ZWrote out file "/tmp/test_repair_verify.data"
120942024-01-24T19:10:38.511Z
120952024-01-24T19:10:38.515ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
120962024-01-24T19:10:38.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120972024-01-24T19:10:38.528Z 0 000-019 99 99 99 171 171 171 F F F
120982024-01-24T19:10:38.534Z 1 020-039 110 110 110 187 187 187 F F F
120992024-01-24T19:10:38.534Z 2 040-059 109 109 109 185 185 185 F F F
121002024-01-24T19:10:38.534Z 3 060-079 104 104 104 179 179 179 F F F
121012024-01-24T19:10:38.534Z 4 080-099 109 109 109 185 185 185 F F F
121022024-01-24T19:10:38.534Z 5 100-119 108 108 108 182 182 182 F F F
121032024-01-24T19:10:38.534Z 6 120-139 110 110 110 187 187 187 T T F <---
121042024-01-24T19:10:38.534Z 7 140-159 111 111 111 189 189 189 F F F
121052024-01-24T19:10:38.534Z 8 160-179 109 109 109 185 185 185 F F F
121062024-01-24T19:10:38.534Z 9 180-199 110 110 110 187 187 187 F F F
121072024-01-24T19:10:38.534Z 10 200-219 110 110 110 186 186 186 F F F
121082024-01-24T19:10:38.535Z 11 220-239 112 112 110 194 194 188 T T F <---
121092024-01-24T19:10:38.535Z 12 240-259 112 112 112 194 194 190 F F F <---
121102024-01-24T19:10:38.535Z 13 260-279 112 112 109 192 192 185 F F F <---
121112024-01-24T19:10:38.535Z 14 280-299 112 112 110 194 194 187 F F F <---
121122024-01-24T19:10:38.535Z 15 300-319 112 112 108 194 194 184 F F F <---
121132024-01-24T19:10:38.535Z 16 320-339 110 110 110 187 187 187 F F F
121142024-01-24T19:10:38.535Z 17 340-359 108 108 108 182 182 182 F F F
121152024-01-24T19:10:38.535Z 18 360-379 112 112 108 194 194 182 F F F <---
121162024-01-24T19:10:38.535Z 19 380-399 110 110 110 187 187 187 F F F
121172024-01-24T19:10:38.535Z 20 400-419 110 110 110 187 187 187 F F F
121182024-01-24T19:10:38.535Z 21 420-439 100 100 100 173 173 173 F F F
121192024-01-24T19:10:38.535Z 22 440-459 92 92 92 155 155 155 F F F
121202024-01-24T19:10:38.535Z 23 460-479 110 110 110 188 188 188 F F F
121212024-01-24T19:10:38.536Z 24 480-499 112 112 109 194 194 185 F F F <---
121222024-01-24T19:10:38.536Z 25 500-519 109 109 109 185 185 185 F F F
121232024-01-24T19:10:38.536Z 26 520-539 112 112 104 192 192 179 F F T <---
121242024-01-24T19:10:38.536Z 27 540-559 107 107 107 181 181 181 F F F
121252024-01-24T19:10:38.536Z 28 560-579 110 110 110 188 188 188 F F F
121262024-01-24T19:10:38.536Z 29 580-599 112 112 110 194 194 188 F F F <---
121272024-01-24T19:10:38.536ZMax gen: 112, Max flush: 194
121282024-01-24T19:10:38.536ZError: Difference in extent metadata found!
121292024-01-24T19:10:38.540ZA Difference in extent metadata is expected here
121302024-01-24T19:10:38.540ZOn loop 56
121312024-01-24T19:10:38.540Z
121322024-01-24T19:10:38.540ZVerifying data now
121332024-01-24T19:10:43.460ZNo change after verify
121342024-01-24T19:10:43.460ZLoop: 56 Downstairs dump after verify (and repair):
121352024-01-24T19:10:43.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121362024-01-24T19:10:43.606Z 0 000-019 99 99 99 171 171 171 F F F
121372024-01-24T19:10:43.606Z 1 020-039 110 110 110 187 187 187 F F F
121382024-01-24T19:10:43.606Z 2 040-059 109 109 109 185 185 185 F F F
121392024-01-24T19:10:43.606Z 3 060-079 104 104 104 179 179 179 F F F
121402024-01-24T19:10:43.606Z 4 080-099 109 109 109 185 185 185 F F F
121412024-01-24T19:10:43.734Z 5 100-119 108 108 108 182 182 182 F F F
121422024-01-24T19:10:43.734Z 6 120-139 113 113 113 195 195 195 F F F
121432024-01-24T19:10:43.734Z 7 140-159 111 111 111 189 189 189 F F F
121442024-01-24T19:10:43.734Z 8 160-179 109 109 109 185 185 185 F F F
121452024-01-24T19:10:43.735Z 9 180-199 110 110 110 187 187 187 F F F
121462024-01-24T19:10:43.735Z 10 200-219 110 110 110 186 186 186 F F F
121472024-01-24T19:10:43.735Z 11 220-239 113 113 113 195 195 195 F F F
121482024-01-24T19:10:43.735Z 12 240-259 112 112 112 194 194 194 F F F
121492024-01-24T19:10:43.735Z 13 260-279 112 112 112 192 192 192 F F F
121502024-01-24T19:10:43.735Z 14 280-299 112 112 112 194 194 194 F F F
121512024-01-24T19:10:43.736Z 15 300-319 112 112 112 194 194 194 F F F
121522024-01-24T19:10:43.736Z 16 320-339 110 110 110 187 187 187 F F F
121532024-01-24T19:10:43.736Z 17 340-359 108 108 108 182 182 182 F F F
121542024-01-24T19:10:43.736Z 18 360-379 112 112 112 194 194 194 F F F
121552024-01-24T19:10:43.736Z 19 380-399 110 110 110 187 187 187 F F F
121562024-01-24T19:10:43.736Z 20 400-419 110 110 110 187 187 187 F F F
121572024-01-24T19:10:43.736Z 21 420-439 100 100 100 173 173 173 F F F
121582024-01-24T19:10:43.736Z 22 440-459 92 92 92 155 155 155 F F F
121592024-01-24T19:10:43.738Z 23 460-479 110 110 110 188 188 188 F F F
121602024-01-24T19:10:43.738Z 24 480-499 112 112 112 194 194 194 F F F
121612024-01-24T19:10:43.738Z 25 500-519 109 109 109 185 185 185 F F F
121622024-01-24T19:10:43.738Z 26 520-539 112 112 112 192 192 192 F F F
121632024-01-24T19:10:43.738Z 27 540-559 107 107 107 181 181 181 F F F
121642024-01-24T19:10:43.738Z 28 560-579 110 110 110 188 188 188 F F F
121652024-01-24T19:10:43.738Z 29 580-599 112 112 112 194 194 194 F F F
121662024-01-24T19:10:43.738ZMax gen: 113, Max flush: 195
121672024-01-24T19:10:43.738Z
121682024-01-24T19:10:43.738ZBegin loop 57
121692024-01-24T19:10:43.738ZDownstairs to restart: 0
121702024-01-24T19:10:43.739Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.187892946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585}
121712024-01-24T19:10:43.739Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188426193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585}
121722024-01-24T19:10:43.739Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188457042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585}
121732024-01-24T19:10:43.739Z{"msg":"Crucible 00802e07-2cc7-4c4e-852e-3007fb9cdfc0 has session id: 37ffc037-c068-4b0d-b0c2-d19e2232c079","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.18850654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121742024-01-24T19:10:43.739Z{"msg":"Upstairs opts: Upstairs UUID: 00802e07-2cc7-4c4e-852e-3007fb9cdfc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188546222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121752024-01-24T19:10:43.739Z{"msg":"Crucible stats registered with UUID: 00802e07-2cc7-4c4e-852e-3007fb9cdfc0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188617804Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121762024-01-24T19:10:43.739ZCrucible runtime is spawned
121772024-01-24T19:10:43.739Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188881671Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585}
121782024-01-24T19:10:43.739Z{"msg":"00802e07-2cc7-4c4e-852e-3007fb9cdfc0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.188994572Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121792024-01-24T19:10:43.739Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.18907082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121802024-01-24T19:10:43.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189172483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121812024-01-24T19:10:43.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189205748Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121822024-01-24T19:10:43.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189279146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121832024-01-24T19:10:43.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189306713Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121842024-01-24T19:10:43.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189329973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121852024-01-24T19:10:43.740Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189369699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121862024-01-24T19:10:43.740Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189474459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121872024-01-24T19:10:43.740Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189552029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121882024-01-24T19:10:43.740Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189629442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121892024-01-24T19:10:43.740Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.1896733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121902024-01-24T19:10:43.740Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.189713919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121912024-01-24T19:10:43.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.190879367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121922024-01-24T19:10:43.740Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.190974648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121932024-01-24T19:10:43.740Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.191117474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121942024-01-24T19:10:43.740Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19116057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121952024-01-24T19:10:43.741Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.192052983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121962024-01-24T19:10:43.741Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.192243204Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121972024-01-24T19:10:43.741Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.192308809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121982024-01-24T19:10:43.741Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.192336459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
121992024-01-24T19:10:43.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19304089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122002024-01-24T19:10:43.741Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.193113873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122012024-01-24T19:10:43.741Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.193142003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122022024-01-24T19:10:43.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19322049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122032024-01-24T19:10:43.741Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19324949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122042024-01-24T19:10:43.741Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.193308129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122052024-01-24T19:10:43.742Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.193477444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122062024-01-24T19:10:43.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194179721Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122072024-01-24T19:10:43.743Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194249415Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122082024-01-24T19:10:43.743Z{"msg":"[0]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 195, 189, 185, 187, 186, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194297587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122092024-01-24T19:10:43.743Z{"msg":"[0]R generation[0..12]: [99, 110, 109, 104, 109, 108, 113, 111, 109, 110, 110, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194337975Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122102024-01-24T19:10:43.743Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194406221Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122112024-01-24T19:10:43.743Z{"msg":"[1]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 195, 189, 185, 187, 186, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194465126Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122122024-01-24T19:10:43.743Z{"msg":"[1]R generation[0..12]: [99, 110, 109, 104, 109, 108, 113, 111, 109, 110, 110, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19449205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122132024-01-24T19:10:43.743Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194515895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122142024-01-24T19:10:43.745Z{"msg":"[2]R flush_numbers[0..12]: [171, 187, 185, 179, 185, 182, 195, 189, 185, 187, 186, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194544746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122152024-01-24T19:10:43.745Z{"msg":"[2]R generation[0..12]: [99, 110, 109, 104, 109, 108, 113, 111, 109, 110, 110, 113]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
122162024-01-24T19:10:43.745Z,"time":"2024-01-24T19:10:42.194611738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122172024-01-24T19:10:43.745Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194660872Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122182024-01-24T19:10:43.745Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.19468639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122192024-01-24T19:10:43.745Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194709025Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122202024-01-24T19:10:43.745Z{"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194769032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122212024-01-24T19:10:43.745Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194797258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122222024-01-24T19:10:43.746Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194856908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122232024-01-24T19:10:43.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194882234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122242024-01-24T19:10:43.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194947145Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122252024-01-24T19:10:43.746Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.194979307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122262024-01-24T19:10:43.746Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.195077644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122272024-01-24T19:10:43.746Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.195104198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122282024-01-24T19:10:43.746Z{"msg":"00802e07-2cc7-4c4e-852e-3007fb9cdfc0 is now active with session: 37ffc037-c068-4b0d-b0c2-d19e2232c079","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.195125554Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122292024-01-24T19:10:43.746Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.195146675Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585,"session_id":"37ffc037-c068-4b0d-b0c2-d19e2232c079"}
122302024-01-24T19:10:43.746Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:42.195199242Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1585}
122312024-01-24T19:10:43.746ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122322024-01-24T19:10:43.746ZLoading write count information from file "/tmp/test_repair_verify.data"
122332024-01-24T19:10:43.746ZRun Repair workload
122342024-01-24T19:10:43.746Z01/30 Write block 41 len 28672 data: 6 5 6 10 13 10 9
122352024-01-24T19:10:43.746Z02/30 Read block 554 len 8192
122362024-01-24T19:10:43.747Z03/30 Read block 533 len 24576
122372024-01-24T19:10:43.747Z04/30 Read block 12 len 4096
122382024-01-24T19:10:43.747Z05/30 Write block 362 len 16384 data: 12 11 13 13
122392024-01-24T19:10:43.747Z06/30 Write block 273 len 4096 data: 12
122402024-01-24T19:10:43.748Z07/30 Read block 502 len 36864
122412024-01-24T19:10:43.763Z08/30 Read block 450 len 36864
122422024-01-24T19:10:43.763Z09/30 Write block 91 len 16384 data: 8 9 11 12
122432024-01-24T19:10:43.763Z10/30 Read block 169 len 4096
122442024-01-24T19:10:43.763Z11/30 Write block 196 len 40960 data: 8 8 7 5 5 6 5 6 8 8
122452024-01-24T19:10:43.763Z12/30 Flush
122462024-01-24T19:10:43.763Z13/30 Flush
122472024-01-24T19:10:43.764Z14/30 Write block 430 len 4096 data: 11
122482024-01-24T19:10:43.764Z15/30 Read block 125 len 24576
122492024-01-24T19:10:43.764Z16/30 Write block 107 len 24576 data: 10 11 9 8 7 7
122502024-01-24T19:10:43.764Z17/30 Read block 221 len 32768
122512024-01-24T19:10:43.764Z18/30 Write block 496 len 20480 data: 13 12 13 12 9
122522024-01-24T19:10:43.764Z19/30 Flush
122532024-01-24T19:10:43.764Z20/30 Write block 154 len 16384 data: 11 9 9 11
122542024-01-24T19:10:43.765Z21/30 Write block 528 len 4096 data: 14
122552024-01-24T19:10:43.765Z22/30 Read block 190 len 24576
122562024-01-24T19:10:43.765Z23/30 Write block 567 len 4096 data: 10
122572024-01-24T19:10:43.765Z24/30 Write block 14 len 32768 data: 5 7 4 5 5 7 7 9
122582024-01-24T19:10:43.765Z25/30 Read block 1 len 16384
122592024-01-24T19:10:43.765Z26/30 Read block 346 len 8192
122602024-01-24T19:10:43.766Z27/30 Write block 375 len 40960 data: 11 10 10 7 7 10 9 7 8 6
122612024-01-24T19:10:43.766Z28/30 Read block 56 len 4096
122622024-01-24T19:10:43.766Z29/30 Read block 58 len 24576
122632024-01-24T19:10:43.766Z30/30 Read block 514 len 36864
122642024-01-24T19:10:43.766Z----------------------------------------------------------------
122652024-01-24T19:10:43.767Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
122662024-01-24T19:10:43.767ZStates: Active Active Active
122672024-01-24T19:10:43.768ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122682024-01-24T19:10:43.768Z 1 Acked 1000 Write 7 Done Done Done false
122692024-01-24T19:10:43.768Z 2 Acked 1001 Read 2 Done Done Done false
122702024-01-24T19:10:43.768Z 3 Acked 1002 Read 6 Done Done Done false
122712024-01-24T19:10:43.768Z 4 Acked 1003 Read 1 Done Done Done false
122722024-01-24T19:10:43.768Z 5 Acked 1004 Write 4 Sent Done Done false
122732024-01-24T19:10:43.768Z 6 Acked 1005 Write 1 Sent Done Done false
122742024-01-24T19:10:43.768Z 7 Acked 1006 Read 9 Sent Done Done false
122752024-01-24T19:10:43.769Z 8 Acked 1007 Read 9 Sent Done Done false
122762024-01-24T19:10:43.769Z 9 Acked 1008 Write 4 Sent Done Done false
122772024-01-24T19:10:43.769Z 10 Acked 1009 Read 1 Sent Done Done false
122782024-01-24T19:10:43.769Z 11 Acked 1010 Write 10 Sent Done Done false
122792024-01-24T19:10:43.769Z 12 Acked 1011 Flush 0 Sent Done Done false
122802024-01-24T19:10:43.769Z 13 Acked 1012 Flush 0 Sent Done Done false
122812024-01-24T19:10:43.769Z 14 Acked 1013 Write 1 Sent Done Done false
122822024-01-24T19:10:43.769Z 15 Acked 1014 Read 6 Sent Done Done false
122832024-01-24T19:10:43.769Z 16 Acked 1015 Write 6 Sent Done Done false
122842024-01-24T19:10:43.770Z 17 Acked 1016 Read 8 Sent Done Done false
122852024-01-24T19:10:43.770Z 18 Acked 1017 Write 5 Sent Done Done false
122862024-01-24T19:10:43.770Z 19 Acked 1018 Flush 0 Sent Done Done false
122872024-01-24T19:10:43.770Z 20 Acked 1019 Write 4 Sent Done Done false
122882024-01-24T19:10:43.771Z 21 Acked 1020 Write 1 Sent Done Done false
122892024-01-24T19:10:43.771Z 22 Acked 1021 Read 6 Sent Done Done false
122902024-01-24T19:10:43.771Z 23 Acked 1022 Write 1 Sent Done Done false
122912024-01-24T19:10:43.771Z 24 Acked 1023 Write 8 Sent Done Done false
122922024-01-24T19:10:43.771Z 25 Acked 1024 Read 4 Sent Done Done false
122932024-01-24T19:10:43.771Z 26 Acked 1025 Read 2 Sent Done Done false
122942024-01-24T19:10:43.771Z 27 Acked 1026 Write 10 Sent Done Done false
122952024-01-24T19:10:43.772Z 28 Acked 1027 Read 1 Sent Done Done false
122962024-01-24T19:10:43.772Z 29 Acked 1028 Read 6 Sent Done Done false
122972024-01-24T19:10:43.772Z 30 Acked 1029 Read 9 Sent Sent Done false
122982024-01-24T19:10:43.772Z STATES DS:0 DS:1 DS:2 TOTAL
122992024-01-24T19:10:43.772Z New 0 0 0 0
123002024-01-24T19:10:43.772Z Sent 26 1 0 27
123012024-01-24T19:10:43.772Z Done 4 29 30 63
123022024-01-24T19:10:43.772Z Skipped 0 0 0 0
123032024-01-24T19:10:43.772Z Error 0 0 0 0
123042024-01-24T19:10:43.773ZLast Flush: 0 1018 1018
123052024-01-24T19:10:43.773ZDownstairs last five completed:
123062024-01-24T19:10:43.773ZUpstairs last five completed: 30 29 28 27 26
123072024-01-24T19:10:43.773ZWrote out file "/tmp/test_repair_verify.data"
123082024-01-24T19:10:43.773Z
123092024-01-24T19:10:43.773ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123102024-01-24T19:10:43.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123112024-01-24T19:10:43.774Z 0 000-019 99 99 99 171 171 171 F T T <---
123122024-01-24T19:10:43.774Z 1 020-039 110 110 110 187 187 187 F T T <---
123132024-01-24T19:10:43.774Z 2 040-059 109 114 114 185 196 196 T F F <---
123142024-01-24T19:10:43.774Z 3 060-079 104 104 104 179 179 179 F F F
123152024-01-24T19:10:43.774Z 4 080-099 109 114 114 185 196 196 F F F <---
123162024-01-24T19:10:43.774Z 5 100-119 108 114 114 182 198 198 F F F <---
123172024-01-24T19:10:43.774Z 6 120-139 113 113 113 195 195 195 F F F
123182024-01-24T19:10:43.775Z 7 140-159 111 111 111 189 189 189 F T T <---
123192024-01-24T19:10:43.775Z 8 160-179 109 109 109 185 185 185 F F F
123202024-01-24T19:10:43.775Z 9 180-199 110 114 114 187 196 196 F F F <---
123212024-01-24T19:10:43.775Z 10 200-219 110 114 114 186 196 196 F F F <---
123222024-01-24T19:10:43.775Z 11 220-239 113 113 113 195 195 195 F F F
123232024-01-24T19:10:43.775Z 12 240-259 112 112 112 194 194 194 F F F
123242024-01-24T19:10:43.775Z 13 260-279 112 114 114 192 196 196 F F F <---
123252024-01-24T19:10:43.775Z 14 280-299 112 112 112 194 194 194 F F F
123262024-01-24T19:10:43.776Z 15 300-319 112 112 112 194 194 194 F F F
123272024-01-24T19:10:43.776Z 16 320-339 110 110 110 187 187 187 F F F
123282024-01-24T19:10:43.776Z 17 340-359 108 108 108 182 182 182 F F F
123292024-01-24T19:10:43.776Z 18 360-379 112 114 114 194 196 196 F T T <---
123302024-01-24T19:10:43.776Z 19 380-399 110 110 110 187 187 187 F T T <---
123312024-01-24T19:10:43.776Z 20 400-419 110 110 110 187 187 187 F F F
123322024-01-24T19:10:43.776Z 21 420-439 100 114 114 173 198 198 F F F <---
123332024-01-24T19:10:43.776Z 22 440-459 92 92 92 155 155 155 F F F
123342024-01-24T19:10:43.776Z 23 460-479 110 110 110 188 188 188 F F F
123352024-01-24T19:10:43.777Z 24 480-499 112 114 114 194 198 198 F F F <---
123362024-01-24T19:10:43.778Z 25 500-519 109 114 114 185 198 198 F F F <---
123372024-01-24T19:10:43.778Z 26 520-539 112 112 112 192 192 192 F T T <---
123382024-01-24T19:10:43.778Z 27 540-559 107 107 107 181 181 181 F F F
123392024-01-24T19:10:43.778Z 28 560-579 110 110 110 188 188 188 F T T <---
123402024-01-24T19:10:43.779Z 29 580-599 112 112 112 194 194 194 F F F
123412024-01-24T19:10:43.867ZMax gen: 114, Max flush: 198
123422024-01-24T19:10:43.868ZError: Difference in extent metadata found!
123432024-01-24T19:10:43.868ZA Difference in extent metadata is expected here
123442024-01-24T19:10:43.868ZOn loop 57
123452024-01-24T19:10:43.868Z
123462024-01-24T19:10:43.868ZVerifying data now
123472024-01-24T19:10:49.031ZNo change after verify
123482024-01-24T19:10:49.032ZLoop: 57 Downstairs dump after verify (and repair):
123492024-01-24T19:10:49.044ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123502024-01-24T19:10:49.044Z 0 000-019 115 115 115 199 199 199 F F F
123512024-01-24T19:10:49.044Z 1 020-039 115 115 115 199 199 199 F F F
123522024-01-24T19:10:49.044Z 2 040-059 114 114 114 196 196 196 F F F
123532024-01-24T19:10:49.044Z 3 060-079 104 104 104 179 179 179 F F F
123542024-01-24T19:10:49.044Z 4 080-099 114 114 114 196 196 196 F F F
123552024-01-24T19:10:49.045Z 5 100-119 114 114 114 198 198 198 F F F
123562024-01-24T19:10:49.045Z 6 120-139 113 113 113 195 195 195 F F F
123572024-01-24T19:10:49.045Z 7 140-159 115 115 115 199 199 199 F F F
123582024-01-24T19:10:49.045Z 8 160-179 109 109 109 185 185 185 F F F
123592024-01-24T19:10:49.045Z 9 180-199 114 114 114 196 196 196 F F F
123602024-01-24T19:10:49.049Z 10 200-219 114 114 114 196 196 196 F F F
123612024-01-24T19:10:49.049Z 11 220-239 113 113 113 195 195 195 F F F
123622024-01-24T19:10:49.049Z 12 240-259 112 112 112 194 194 194 F F F
123632024-01-24T19:10:49.049Z 13 260-279 114 114 114 196 196 196 F F F
123642024-01-24T19:10:49.050Z 14 280-299 112 112 112 194 194 194 F F F
123652024-01-24T19:10:49.050Z 15 300-319 112 112 112 194 194 194 F F F
123662024-01-24T19:10:49.050Z 16 320-339 110 110 110 187 187 187 F F F
123672024-01-24T19:10:49.050Z 17 340-359 108 108 108 182 182 182 F F F
123682024-01-24T19:10:49.050Z 18 360-379 115 115 115 199 199 199 F F F
123692024-01-24T19:10:49.050Z 19 380-399 115 115 115 199 199 199 F F F
123702024-01-24T19:10:49.050Z 20 400-419 110 110 110 187 187 187 F F F
123712024-01-24T19:10:49.050Z 21 420-439 114 114 114 198 198 198 F F F
123722024-01-24T19:10:49.050Z 22 440-459 92 92 92 155 155 155 F F F
123732024-01-24T19:10:49.050Z 23 460-479 110 110 110 188 188 188 F F F
123742024-01-24T19:10:49.050Z 24 480-499 114 114 114 198 198 198 F F F
123752024-01-24T19:10:49.050Z 25 500-519 114 114 114 198 198 198 F F F
123762024-01-24T19:10:49.051Z 26 520-539 115 115 115 199 199 199 F F F
123772024-01-24T19:10:49.051Z 27 540-559 107 107 107 181 181 181 F F F
123782024-01-24T19:10:49.051Z 28 560-579 115 115 115 199 199 199 F F F
123792024-01-24T19:10:49.051Z 29 580-599 112 112 112 194 194 194 F F F
123802024-01-24T19:10:49.051ZMax gen: 115, Max flush: 199
123812024-01-24T19:10:49.055Z
123822024-01-24T19:10:49.055ZBegin loop 58
123832024-01-24T19:10:49.055ZDownstairs to restart: 2
123842024-01-24T19:10:49.088Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.752143885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593}
123852024-01-24T19:10:49.088Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.752672828Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593}
123862024-01-24T19:10:49.088Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.752755311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593}
123872024-01-24T19:10:49.097Z{"msg":"Crucible acaa49b6-2fed-4d2a-8882-1b6c9c9ee5e8 has session id: 45ecd644-9721-4887-b377-c3782ca39156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.752794296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123882024-01-24T19:10:49.097Z{"msg":"Crucible runtime is spawned
123892024-01-24T19:10:49.097ZUpstairs opts: Upstairs UUID: acaa49b6-2fed-4d2a-8882-1b6c9c9ee5e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.752943049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123902024-01-24T19:10:49.097Z{"msg":"Crucible stats registered with UUID: acaa49b6-2fed-4d2a-8882-1b6c9c9ee5e8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.75305927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123912024-01-24T19:10:49.097Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.75312983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593}
123922024-01-24T19:10:49.097Z{"msg":"acaa49b6-2fed-4d2a-8882-1b6c9c9ee5e8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.75328446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123932024-01-24T19:10:49.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753344355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123942024-01-24T19:10:49.097Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753372752Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123952024-01-24T19:10:49.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753403405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123962024-01-24T19:10:49.098Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753441481Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123972024-01-24T19:10:49.098Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753475679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123982024-01-24T19:10:49.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753596855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
123992024-01-24T19:10:49.099Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753640827Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124002024-01-24T19:10:49.099Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753717461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124012024-01-24T19:10:49.099Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753762504Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124022024-01-24T19:10:49.099Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753877374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124032024-01-24T19:10:49.099Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.753958863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124042024-01-24T19:10:49.099Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.754078284Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124052024-01-24T19:10:49.099Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.755375925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124062024-01-24T19:10:49.099Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.755426384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124072024-01-24T19:10:49.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.755470055Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124082024-01-24T19:10:49.101Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.755582454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124092024-01-24T19:10:49.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.756441408Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124102024-01-24T19:10:49.101Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.756526592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124112024-01-24T19:10:49.101Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.756607159Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124122024-01-24T19:10:49.101Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.756635612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124132024-01-24T19:10:49.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757547623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124142024-01-24T19:10:49.101Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757642192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124152024-01-24T19:10:49.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757688788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124162024-01-24T19:10:49.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757748312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124172024-01-24T19:10:49.102Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757793332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124182024-01-24T19:10:49.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757855172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124192024-01-24T19:10:49.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.757897296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124202024-01-24T19:10:49.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758574912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124212024-01-24T19:10:49.102Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.75869645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124222024-01-24T19:10:49.102Z{"msg":"[0]R flush_numbers[0..12]: [199, 199, 196, 179, 196, 198, 195, 199, 185, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758759851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124232024-01-24T19:10:49.102Z{"msg":"[0]R generation[0..12]: [115, 115, 114, 104, 114, 114, 113, 115, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758788238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124242024-01-24T19:10:49.102Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.75883343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124252024-01-24T19:10:49.103Z{"msg":"[1]R flush_numbers[0..12]: [199, 199, 196, 179, 196, 198, 195, 199, 185, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758860448Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124262024-01-24T19:10:49.104Z{"msg":"[1]R generation[0..12]: [115, 115, 114, 104, 114, 114, 113, 115, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758883559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124272024-01-24T19:10:49.104Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
124282024-01-24T19:10:49.104Zcrucible","level":30,"time":"2024-01-24T19:10:47.758911077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124292024-01-24T19:10:49.104Z{"msg":"[2]R flush_numbers[0..12]: [199, 199, 196, 179, 196, 198, 195, 199, 185, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758956212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124302024-01-24T19:10:49.104Z{"msg":"[2]R generation[0..12]: [115, 115, 114, 104, 114, 114, 113, 115, 109, 114, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.758982561Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124312024-01-24T19:10:49.104Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759062349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124322024-01-24T19:10:49.104Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759089289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124332024-01-24T19:10:49.104Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759139427Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124342024-01-24T19:10:49.104Z{"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759163771Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124352024-01-24T19:10:49.104Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759213243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124362024-01-24T19:10:49.107Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759238446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124372024-01-24T19:10:49.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759279576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124382024-01-24T19:10:49.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759304256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124392024-01-24T19:10:49.107Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759347395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124402024-01-24T19:10:49.108Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759372345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124412024-01-24T19:10:49.108Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759417472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124422024-01-24T19:10:49.108Z{"msg":"acaa49b6-2fed-4d2a-8882-1b6c9c9ee5e8 is now active with session: 45ecd644-9721-4887-b377-c3782ca39156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759442278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124432024-01-24T19:10:49.108Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759483593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593,"session_id":"45ecd644-9721-4887-b377-c3782ca39156"}
124442024-01-24T19:10:49.108Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:47.759506993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1593}
124452024-01-24T19:10:49.108ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124462024-01-24T19:10:49.108ZLoading write count information from file "/tmp/test_repair_verify.data"
124472024-01-24T19:10:49.108ZRun Repair workload
124482024-01-24T19:10:49.108Z01/30 Write block 244 len 16384 data: 13 13 15 14
124492024-01-24T19:10:49.113Z02/30 Read block 578 len 24576
124502024-01-24T19:10:49.113Z03/30 Write block 63 len 8192 data: 8 10
124512024-01-24T19:10:49.113Z04/30 Read block 35 len 28672
124522024-01-24T19:10:49.113Z05/30 Read block 51 len 20480
124532024-01-24T19:10:49.113Z06/30 Read block 342 len 36864
124542024-01-24T19:10:49.114Z07/30 Read block 191 len 8192
124552024-01-24T19:10:49.114Z08/30 Write block 404 len 20480 data: 5 8 9 9 10
124562024-01-24T19:10:49.114Z09/30 Write block 563 len 40960 data: 5 5 8 8 11 12 11 7 7 8
124572024-01-24T19:10:49.114Z10/30 Read block 343 len 12288
124582024-01-24T19:10:49.114Z11/30 Read block 456 len 16384
124592024-01-24T19:10:49.114Z12/30 Read block 15 len 24576
124602024-01-24T19:10:49.114Z13/30 Read block 198 len 32768
124612024-01-24T19:10:49.114Z14/30 Read block 239 len 28672
124622024-01-24T19:10:49.122Z15/30 Write block 177 len 32768 data: 8 9 7 9 8 7 7 9
124632024-01-24T19:10:49.122Z16/30 Read block 44 len 28672
124642024-01-24T19:10:49.123Z17/30 Flush
124652024-01-24T19:10:49.138Z18/30 Flush
124662024-01-24T19:10:49.139Z19/30 Write block 141 len 36864 data: 8 4 4 6 7 9 11 9 11
124672024-01-24T19:10:49.139Z20/30 Read block 424 len 8192
124682024-01-24T19:10:49.139Z21/30 Read block 420 len 4096
124692024-01-24T19:10:49.139Z22/30 Read block 186 len 12288
124702024-01-24T19:10:49.139Z23/30 Write block 311 len 24576 data: 6 7 10 11 12 13
124712024-01-24T19:10:49.139Z24/30 Write block 562 len 8192 data: 6 6
124722024-01-24T19:10:49.139Z25/30 Write block 167 len 20480 data: 11 13 11 10 11
124732024-01-24T19:10:49.150Z26/30 Read block 409 len 8192
124742024-01-24T19:10:49.151Z27/30 Write block 347 len 40960 data: 6 6 9 7 9 8 9 8 9 11
124752024-01-24T19:10:49.151Z28/30 Write block 542 len 28672 data: 11 11 13 11 11 12 10
124762024-01-24T19:10:49.151Z29/30 Write block 172 len 12288 data: 11 9 11
124772024-01-24T19:10:49.152Z30/30 Read block 403 len 16384
124782024-01-24T19:10:49.152Z----------------------------------------------------------------
124792024-01-24T19:10:49.152Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
124802024-01-24T19:10:49.152ZStates: Active Active Active
124812024-01-24T19:10:49.152ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124822024-01-24T19:10:49.152Z 1 Acked 1000 Write 4 Done Done Done false
124832024-01-24T19:10:49.152Z 2 Acked 1001 Read 6 Done Done Done false
124842024-01-24T19:10:49.153Z 3 Acked 1002 Write 2 Done Done Done false
124852024-01-24T19:10:49.153Z 4 Acked 1003 Read 7 Done Done Done false
124862024-01-24T19:10:49.153Z 5 Acked 1004 Read 5 Done Done Done false
124872024-01-24T19:10:49.153Z 6 Acked 1005 Read 9 Done Done Done false
124882024-01-24T19:10:49.153Z 7 Acked 1006 Read 2 Done Done Done false
124892024-01-24T19:10:49.153Z 8 Acked 1007 Write 5 Done Done Sent false
124902024-01-24T19:10:49.154Z 9 Acked 1008 Write 10 Done Done Sent false
124912024-01-24T19:10:49.154Z 10 Acked 1009 Read 3 Done Done Sent false
124922024-01-24T19:10:49.154Z 11 Acked 1010 Read 4 Done Done Sent false
124932024-01-24T19:10:49.154Z 12 Acked 1011 Read 6 Done Done Sent false
124942024-01-24T19:10:49.154Z 13 Acked 1012 Read 8 Done Done Sent false
124952024-01-24T19:10:49.154Z 14 Acked 1013 Read 7 Done Done Sent false
124962024-01-24T19:10:49.155Z 15 Acked 1014 Write 8 Done Done Sent false
124972024-01-24T19:10:49.155Z 16 Acked 1015 Read 7 Done Done Sent false
124982024-01-24T19:10:49.155Z 17 Acked 1016 Flush 0 Done Done Sent false
124992024-01-24T19:10:49.155Z 18 Acked 1017 Flush 0 Done Done Sent false
125002024-01-24T19:10:49.155Z 19 Acked 1018 Write 9 Done Done Sent false
125012024-01-24T19:10:49.155Z 20 Acked 1019 Read 2 Done Done Sent false
125022024-01-24T19:10:49.156Z 21 Acked 1020 Read 1 Done Done Sent false
125032024-01-24T19:10:49.157Z 22 Acked 1021 Read 3 Done Done Sent false
125042024-01-24T19:10:49.158Z 23 Acked 1022 Write 6 Done Done Sent false
125052024-01-24T19:10:49.158Z 24 Acked 1023 Write 2 Done Done Sent false
125062024-01-24T19:10:49.158Z 25 Acked 1024 Write 5 Done Done Sent false
125072024-01-24T19:10:49.158Z 26 Acked 1025 Read 2 Done Done Sent false
125082024-01-24T19:10:49.159Z 27 Acked 1026 Write 10 Done Done Sent false
125092024-01-24T19:10:49.159Z 28 Acked 1027 Write 7 Done Done Sent false
125102024-01-24T19:10:49.159Z 29 Acked 1028 Write 3 Done Done Sent false
125112024-01-24T19:10:49.159Z 30 Acked 1029 Read 4 Done Sent Sent false
125122024-01-24T19:10:49.160Z STATES DS:0 DS:1 DS:2 TOTAL
125132024-01-24T19:10:49.160Z New 0 0 0 0
125142024-01-24T19:10:49.160Z Sent 0 1 23 24
125152024-01-24T19:10:49.160Z Done 30 29 7 66
125162024-01-24T19:10:49.160Z Skipped 0 0 0 0
125172024-01-24T19:10:49.160Z Error 0 0 0 0
125182024-01-24T19:10:49.161ZLast Flush: 1017 1017 0
125192024-01-24T19:10:49.161ZDownstairs last five completed:
125202024-01-24T19:10:49.161ZUpstairs last five completed: 30 29 28 27 26
125212024-01-24T19:10:49.162ZWrote out file "/tmp/test_repair_verify.data"
125222024-01-24T19:10:49.163Z
125232024-01-24T19:10:49.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125242024-01-24T19:10:49.203ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125252024-01-24T19:10:49.203Z 0 000-019 115 115 115 199 199 199 F F F
125262024-01-24T19:10:49.203Z 1 020-039 115 115 115 199 199 199 F F F
125272024-01-24T19:10:49.203Z 2 040-059 114 114 114 196 196 196 F F F
125282024-01-24T19:10:49.203Z 3 060-079 116 116 104 200 200 179 F F T <---
125292024-01-24T19:10:49.203Z 4 080-099 114 114 114 196 196 196 F F F
125302024-01-24T19:10:49.203Z 5 100-119 114 114 114 198 198 198 F F F
125312024-01-24T19:10:49.203Z 6 120-139 113 113 113 195 195 195 F F F
125322024-01-24T19:10:49.203Z 7 140-159 115 115 115 199 199 199 T T F <---
125332024-01-24T19:10:49.204Z 8 160-179 116 116 109 200 200 185 T T F <---
125342024-01-24T19:10:49.204Z 9 180-199 116 116 114 200 200 196 F F F <---
125352024-01-24T19:10:49.204Z 10 200-219 114 114 114 196 196 196 F F F
125362024-01-24T19:10:49.204Z 11 220-239 113 113 113 195 195 195 F F F
125372024-01-24T19:10:49.204Z 12 240-259 116 116 112 200 200 194 F F T <---
125382024-01-24T19:10:49.204Z 13 260-279 114 114 114 196 196 196 F F F
125392024-01-24T19:10:49.204Z 14 280-299 112 112 112 194 194 194 F F F
125402024-01-24T19:10:49.204Z 15 300-319 112 112 112 194 194 194 T T F <---
125412024-01-24T19:10:49.204Z 16 320-339 110 110 110 187 187 187 F F F
125422024-01-24T19:10:49.204Z 17 340-359 108 108 108 182 182 182 T T F <---
125432024-01-24T19:10:49.204Z 18 360-379 115 115 115 199 199 199 F F F
125442024-01-24T19:10:49.204Z 19 380-399 115 115 115 199 199 199 F F F
125452024-01-24T19:10:49.204Z 20 400-419 116 116 110 200 200 187 F F F <---
125462024-01-24T19:10:49.204Z 21 420-439 114 114 114 198 198 198 F F F
125472024-01-24T19:10:49.204Z 22 440-459 92 92 92 155 155 155 F F F
125482024-01-24T19:10:49.204Z 23 460-479 110 110 110 188 188 188 F F F
125492024-01-24T19:10:49.204Z 24 480-499 114 114 114 198 198 198 F F F
125502024-01-24T19:10:49.204Z 25 500-519 114 114 114 198 198 198 F F F
125512024-01-24T19:10:49.205Z 26 520-539 115 115 115 199 199 199 F F F
125522024-01-24T19:10:49.205Z 27 540-559 107 107 107 181 181 181 T T F <---
125532024-01-24T19:10:49.205Z 28 560-579 116 116 115 200 200 199 T T F <---
125542024-01-24T19:10:49.205Z 29 580-599 112 112 112 194 194 194 F F F
125552024-01-24T19:10:49.205ZMax gen: 116, Max flush: 200
125562024-01-24T19:10:49.205ZError: Difference in extent metadata found!
125572024-01-24T19:10:49.205ZA Difference in extent metadata is expected here
125582024-01-24T19:10:49.205ZOn loop 58
125592024-01-24T19:10:49.205Z
125602024-01-24T19:10:49.205ZVerifying data now
125612024-01-24T19:10:55.145ZNo change after verify
125622024-01-24T19:10:55.145ZLoop: 58 Downstairs dump after verify (and repair):
125632024-01-24T19:10:55.168ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125642024-01-24T19:10:55.169Z 0 000-019 115 115 115 199 199 199 F F F
125652024-01-24T19:10:55.172Z 1 020-039 115 115 115 199 199 199 F F F
125662024-01-24T19:10:55.172Z 2 040-059 114 114 114 196 196 196 F F F
125672024-01-24T19:10:55.172Z 3 060-079 116 116 116 200 200 200 F F F
125682024-01-24T19:10:55.172Z 4 080-099 114 114 114 196 196 196 F F F
125692024-01-24T19:10:55.172Z 5 100-119 114 114 114 198 198 198 F F F
125702024-01-24T19:10:55.172Z 6 120-139 113 113 113 195 195 195 F F F
125712024-01-24T19:10:55.172Z 7 140-159 117 117 117 201 201 201 F F F
125722024-01-24T19:10:55.172Z 8 160-179 117 117 117 201 201 201 F F F
125732024-01-24T19:10:55.172Z 9 180-199 116 116 116 200 200 200 F F F
125742024-01-24T19:10:55.172Z 10 200-219 114 114 114 196 196 196 F F F
125752024-01-24T19:10:55.172Z 11 220-239 113 113 113 195 195 195 F F F
125762024-01-24T19:10:55.172Z 12 240-259 116 116 116 200 200 200 F F F
125772024-01-24T19:10:55.172Z 13 260-279 114 114 114 196 196 196 F F F
125782024-01-24T19:10:55.172Z 14 280-299 112 112 112 194 194 194 F F F
125792024-01-24T19:10:55.173Z 15 300-319 117 117 117 201 201 201 F F F
125802024-01-24T19:10:55.173Z 16 320-339 110 110 110 187 187 187 F F F
125812024-01-24T19:10:55.173Z 17 340-359 117 117 117 201 201 201 F F F
125822024-01-24T19:10:55.173Z 18 360-379 115 115 115 199 199 199 F F F
125832024-01-24T19:10:55.173Z 19 380-399 115 115 115 199 199 199 F F F
125842024-01-24T19:10:55.174Z 20 400-419 116 116 116 200 200 200 F F F
125852024-01-24T19:10:55.174Z 21 420-439 114 114 114 198 198 198 F F F
125862024-01-24T19:10:55.174Z 22 440-459 92 92 92 155 155 155 F F F
125872024-01-24T19:10:55.174Z 23 460-479 110 110 110 188 188 188 F F F
125882024-01-24T19:10:55.174Z 24 480-499 114 114 114 198 198 198 F F F
125892024-01-24T19:10:55.174Z 25 500-519 114 114 114 198 198 198 F F F
125902024-01-24T19:10:55.174Z 26 520-539 115 115 115 199 199 199 F F F
125912024-01-24T19:10:55.174Z 27 540-559 117 117 117 201 201 201 F F F
125922024-01-24T19:10:55.174Z 28 560-579 117 117 117 201 201 201 F F F
125932024-01-24T19:10:55.174Z 29 580-599 112 112 112 194 194 194 F F F
125942024-01-24T19:10:55.174ZMax gen: 117, Max flush: 201
125952024-01-24T19:10:55.177Z
125962024-01-24T19:10:55.178ZBegin loop 59
125972024-01-24T19:10:55.178ZDownstairs to restart: 1
125982024-01-24T19:10:55.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.874527785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601}
125992024-01-24T19:10:55.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875074184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601}
126002024-01-24T19:10:55.205Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875107534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601}
126012024-01-24T19:10:55.209Z{"msg":"Crucible 6afcd64d-df57-4729-bd5c-42b85e1e4bb9 has session id: 4ead5782-5142-4a7d-a780-2a2f06470c8a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875169401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126022024-01-24T19:10:55.209Z{"msg":"Upstairs opts: Upstairs UUID: 6afcd64d-df57-4729-bd5c-42b85e1e4bb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875235498Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126032024-01-24T19:10:55.211Z{"msg":"Crucible stats registered with UUID: 6afcd64d-df57-4729-bd5c-42b85e1e4bb9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875271666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126042024-01-24T19:10:55.211ZCrucible runtime is spawned
126052024-01-24T19:10:55.211Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875663466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601}
126062024-01-24T19:10:55.211Z{"msg":"6afcd64d-df57-4729-bd5c-42b85e1e4bb9 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.87588187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126072024-01-24T19:10:55.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875950794Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126082024-01-24T19:10:55.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.875998085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126092024-01-24T19:10:55.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876055873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126102024-01-24T19:10:55.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876099982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126112024-01-24T19:10:55.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876159045Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126122024-01-24T19:10:55.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876203877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126132024-01-24T19:10:55.211Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876398971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126142024-01-24T19:10:55.211Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876476897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126152024-01-24T19:10:55.211Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876521728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126162024-01-24T19:10:55.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876612799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126172024-01-24T19:10:55.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.87671043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126182024-01-24T19:10:55.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.876778027Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126192024-01-24T19:10:55.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.877812385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126202024-01-24T19:10:55.212Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.877922747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126212024-01-24T19:10:55.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.877971027Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126222024-01-24T19:10:55.212Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.878043109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126232024-01-24T19:10:55.213Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.878790604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126242024-01-24T19:10:55.218Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879041676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126252024-01-24T19:10:55.218Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879277485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126262024-01-24T19:10:55.218Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879353316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126272024-01-24T19:10:55.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879517555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126282024-01-24T19:10:55.218Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879599937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126292024-01-24T19:10:55.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.87964705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126302024-01-24T19:10:55.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.87991138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126312024-01-24T19:10:55.218Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.879975514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126322024-01-24T19:10:55.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.880003958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126332024-01-24T19:10:55.218Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.880292738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126342024-01-24T19:10:55.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.880999962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126352024-01-24T19:10:55.218Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881066463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126362024-01-24T19:10:55.218Z{"msg":"[0]R flush_numbers[0..12]: [199, 199, 196, 200, 196, 198, 195, 201, 201, 200, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881100442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126372024-01-24T19:10:55.219Z{"msg":"[0]R generation[0..12]: [115, 115, 114, 116, 114, 114, 113, 117, 117, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881160817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126382024-01-24T19:10:55.219Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881194605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126392024-01-24T19:10:55.219Z{"msg":"[1]R flush_numbers[0..12]: [199, 199, 196, 200, 196, 198, 195, 201, 201, 200, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881253697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126402024-01-24T19:10:55.219Z{"msg":"[1]R generation[0..12]: [115, 115, 114, 116, 114, 114, 113, 117, 117, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881286069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126412024-01-24T19:10:55.219Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
126422024-01-24T19:10:55.219Z,"time":"2024-01-24T19:10:53.881345059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126432024-01-24T19:10:55.219Z{"msg":"[2]R flush_numbers[0..12]: [199, 199, 196, 200, 196, 198, 195, 201, 201, 200, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881379863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126442024-01-24T19:10:55.219Z{"msg":"[2]R generation[0..12]: [115, 115, 114, 116, 114, 114, 113, 117, 117, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881441858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126452024-01-24T19:10:55.219Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881475487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126462024-01-24T19:10:55.219Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881532862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126472024-01-24T19:10:55.219Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881560119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126482024-01-24T19:10:55.219Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881619122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126492024-01-24T19:10:55.219Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881650673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126502024-01-24T19:10:55.219Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881708217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126512024-01-24T19:10:55.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881736609Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126522024-01-24T19:10:55.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881797574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126532024-01-24T19:10:55.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881823655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126542024-01-24T19:10:55.219Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881884843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126552024-01-24T19:10:55.219Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881911134Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126562024-01-24T19:10:55.219Z{"msg":"6afcd64d-df57-4729-bd5c-42b85e1e4bb9 is now active with session: 4ead5782-5142-4a7d-a780-2a2f06470c8a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881933122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126572024-01-24T19:10:55.219Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.881952724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601,"session_id":"4ead5782-5142-4a7d-a780-2a2f06470c8a"}
126582024-01-24T19:10:55.220Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:10:53.882012113Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1601}
126592024-01-24T19:10:55.220ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126602024-01-24T19:10:55.220ZLoading write count information from file "/tmp/test_repair_verify.data"
126612024-01-24T19:10:55.220ZRun Repair workload
126622024-01-24T19:10:55.220Z01/30 Write block 194 len 28672 data: 9 8 9 9 8 6 6
126632024-01-24T19:10:55.220Z02/30 Read block 414 len 40960
126642024-01-24T19:10:55.223Z03/30 Read block 91 len 24576
126652024-01-24T19:10:55.223Z04/30 Read block 162 len 24576
126662024-01-24T19:10:55.223Z05/30 Read block 425 len 20480
126672024-01-24T19:10:55.226Z06/30 Read block 262 len 32768
126682024-01-24T19:10:55.226Z07/30 Write block 60 len 28672 data: 8 8 7 9 11 8 5
126692024-01-24T19:10:55.230Z08/30 Read block 540 len 4096
126702024-01-24T19:10:55.230Z09/30 Flush
126712024-01-24T19:10:55.233Z10/30 Write block 343 len 4096 data: 11
126722024-01-24T19:10:55.233Z11/30 Write block 105 len 36864 data: 11 10 11 12 10 9 8 8 6
126732024-01-24T19:10:55.236Z12/30 Write block 579 len 24576 data: 7 7 9 8 9 8
126742024-01-24T19:10:55.236Z13/30 Read block 498 len 12288
126752024-01-24T19:10:55.242Z14/30 Read block 420 len 28672
126762024-01-24T19:10:55.242Z15/30 Read block 304 len 4096
126772024-01-24T19:10:55.242Z16/30 Write block 517 len 40960 data: 14 14 13 11 10 10 8 10 11 12
126782024-01-24T19:10:55.242Z17/30 Write block 152 len 32768 data: 13 12 12 10 10 12 11 7
126792024-01-24T19:10:55.245Z18/30 Read block 20 len 12288
126802024-01-24T19:10:55.245Z19/30 Write block 554 len 32768 data: 7 5 7 9 10 9 7 8
126812024-01-24T19:10:55.248Z20/30 Write block 540 len 4096 data: 7
126822024-01-24T19:10:55.248Z21/30 Read block 555 len 16384
126832024-01-24T19:10:55.248Z22/30 Write block 130 len 24576 data: 9 11 11 9 11 12
126842024-01-24T19:10:55.252Z23/30 Read block 543 len 20480
126852024-01-24T19:10:55.252Z24/30 Write block 358 len 28672 data: 11 10 9 14 13 12 14
126862024-01-24T19:10:55.256Z25/30 Read block 316 len 40960
126872024-01-24T19:10:55.256Z26/30 Write block 31 len 20480 data: 14 12 12 11 6
126882024-01-24T19:10:55.256Z27/30 Read block 482 len 40960
126892024-01-24T19:10:55.260Z28/30 Read block 260 len 28672
126902024-01-24T19:10:55.265Z29/30 Write block 69 len 12288 data: 4 5 4
126912024-01-24T19:10:55.265Z30/30 Read block 465 len 28672
126922024-01-24T19:10:55.266Z----------------------------------------------------------------
126932024-01-24T19:10:55.266Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
126942024-01-24T19:10:55.268ZStates: Active Active Active
126952024-01-24T19:10:55.270ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126962024-01-24T19:10:55.270Z 1 Acked 1000 Write 7 Done Done Done false
126972024-01-24T19:10:55.270Z 2 Acked 1001 Read 10 Done Done Done false
126982024-01-24T19:10:55.270Z 3 Acked 1002 Read 6 Done Done Done false
126992024-01-24T19:10:55.270Z 4 Acked 1003 Read 6 Done Done Done false
127002024-01-24T19:10:55.271Z 5 Acked 1004 Read 5 Done Sent Done false
127012024-01-24T19:10:55.271Z 6 Acked 1005 Read 8 Done Sent Done false
127022024-01-24T19:10:55.271Z 7 Acked 1006 Write 7 Done Sent Done false
127032024-01-24T19:10:55.271Z 8 Acked 1007 Read 1 Done Sent Done false
127042024-01-24T19:10:55.271Z 9 Acked 1008 Flush 0 Done Sent Done false
127052024-01-24T19:10:55.271Z 10 Acked 1009 Write 1 Done Sent Done false
127062024-01-24T19:10:55.271Z 11 Acked 1010 Write 9 Done Sent Done false
127072024-01-24T19:10:55.271Z 12 Acked 1011 Write 6 Done Sent Done false
127082024-01-24T19:10:55.271Z 13 Acked 1012 Read 3 Done Sent Done false
127092024-01-24T19:10:55.271Z 14 Acked 1013 Read 7 Done Sent Done false
127102024-01-24T19:10:55.272Z 15 Acked 1014 Read 1 Done Sent Done false
127112024-01-24T19:10:55.272Z 16 Acked 1015 Write 10 Done Sent Done false
127122024-01-24T19:10:55.272Z 17 Acked 1016 Write 8 Done Sent Done false
127132024-01-24T19:10:55.272Z 18 Acked 1017 Read 3 Done Sent Done false
127142024-01-24T19:10:55.272Z 19 Acked 1018 Write 8 Done Sent Done false
127152024-01-24T19:10:55.272Z 20 Acked 1019 Write 1 Done Sent Done false
127162024-01-24T19:10:55.272Z 21 Acked 1020 Read 4 Done Sent Done false
127172024-01-24T19:10:55.273Z 22 Acked 1021 Write 6 Done Sent Done false
127182024-01-24T19:10:55.273Z 23 Acked 1022 Read 5 Done Sent Done false
127192024-01-24T19:10:55.274Z 24 Acked 1023 Write 7 Done Sent Done false
127202024-01-24T19:10:55.274Z 25 Acked 1024 Read 10 Done Sent Done false
127212024-01-24T19:10:55.274Z 26 Acked 1025 Write 5 Done Sent Done false
127222024-01-24T19:10:55.274Z 27 Acked 1026 Read 10 Done Sent Done false
127232024-01-24T19:10:55.274Z 28 Acked 1027 Read 7 Done Sent Done false
127242024-01-24T19:10:55.274Z 29 Acked 1028 Write 3 Done Sent Done false
127252024-01-24T19:10:55.274Z 30 Acked 1029 Read 7 Done Sent Sent false
127262024-01-24T19:10:55.274Z STATES DS:0 DS:1 DS:2 TOTAL
127272024-01-24T19:10:55.274Z New 0 0 0 0
127282024-01-24T19:10:55.275Z Sent 0 26 1 27
127292024-01-24T19:10:55.275Z Done 30 4 29 63
127302024-01-24T19:10:55.275Z Skipped 0 0 0 0
127312024-01-24T19:10:55.275Z Error 0 0 0 0
127322024-01-24T19:10:55.275ZLast Flush: 1008 0 1008
127332024-01-24T19:10:55.275ZDownstairs last five completed:
127342024-01-24T19:10:55.275ZUpstairs last five completed: 30 29 28 27 26
127352024-01-24T19:10:55.275ZWrote out file "/tmp/test_repair_verify.data"
127362024-01-24T19:10:55.280Z
127372024-01-24T19:10:55.280ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127382024-01-24T19:10:55.311ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127392024-01-24T19:10:55.311Z 0 000-019 115 115 115 199 199 199 F F F
127402024-01-24T19:10:55.311Z 1 020-039 115 115 115 199 199 199 T F T <---
127412024-01-24T19:10:55.312Z 2 040-059 114 114 114 196 196 196 F F F
127422024-01-24T19:10:55.320Z 3 060-079 118 116 118 202 200 202 T F T <---
127432024-01-24T19:10:55.320Z 4 080-099 114 114 114 196 196 196 F F F
127442024-01-24T19:10:55.320Z 5 100-119 114 114 114 198 198 198 T F T <---
127452024-01-24T19:10:55.320Z 6 120-139 113 113 113 195 195 195 T F T <---
127462024-01-24T19:10:55.320Z 7 140-159 117 117 117 201 201 201 T F T <---
127472024-01-24T19:10:55.320Z 8 160-179 117 117 117 201 201 201 F F F
127482024-01-24T19:10:55.320Z 9 180-199 118 116 118 202 200 202 F T F <---
127492024-01-24T19:10:55.320Z 10 200-219 118 114 118 202 196 202 F T F <---
127502024-01-24T19:10:55.321Z 11 220-239 113 113 113 195 195 195 F F F
127512024-01-24T19:10:55.321Z 12 240-259 116 116 116 200 200 200 F F F
127522024-01-24T19:10:55.321Z 13 260-279 114 114 114 196 196 196 F F F
127532024-01-24T19:10:55.321Z 14 280-299 112 112 112 194 194 194 F F F
127542024-01-24T19:10:55.321Z 15 300-319 117 117 117 201 201 201 F F F
127552024-01-24T19:10:55.321Z 16 320-339 110 110 110 187 187 187 F F F
127562024-01-24T19:10:55.321Z 17 340-359 117 117 117 201 201 201 T F T <---
127572024-01-24T19:10:55.321Z 18 360-379 115 115 115 199 199 199 T F T <---
127582024-01-24T19:10:55.321Z 19 380-399 115 115 115 199 199 199 F F F
127592024-01-24T19:10:55.322Z 20 400-419 116 116 116 200 200 200 F F F
127602024-01-24T19:10:55.322Z 21 420-439 114 114 114 198 198 198 F F F
127612024-01-24T19:10:55.322Z 22 440-459 92 92 92 155 155 155 F F F
127622024-01-24T19:10:55.322Z 23 460-479 110 110 110 188 188 188 F F F
127632024-01-24T19:10:55.322Z 24 480-499 114 114 114 198 198 198 F F F
127642024-01-24T19:10:55.322Z 25 500-519 114 114 114 198 198 198 T F T <---
127652024-01-24T19:10:55.322Z 26 520-539 115 115 115 199 199 199 T F T <---
127662024-01-24T19:10:55.323Z 27 540-559 117 117 117 201 201 201 T F T <---
127672024-01-24T19:10:55.323Z 28 560-579 117 117 117 201 201 201 T F T <---
127682024-01-24T19:10:55.323Z 29 580-599 112 112 112 194 194 194 T F T <---
127692024-01-24T19:10:55.323ZMax gen: 118, Max flush: 202
127702024-01-24T19:10:55.323ZError: Difference in extent metadata found!
127712024-01-24T19:10:55.323ZA Difference in extent metadata is expected here
127722024-01-24T19:10:55.323ZOn loop 59
127732024-01-24T19:10:55.323Z
127742024-01-24T19:10:55.324ZVerifying data now
127752024-01-24T19:11:01.575ZNo change after verify
127762024-01-24T19:11:01.575ZLoop: 59 Downstairs dump after verify (and repair):
127772024-01-24T19:11:01.604ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127782024-01-24T19:11:01.604Z 0 000-019 115 115 115 199 199 199 F F F
127792024-01-24T19:11:01.617Z 1 020-039 119 119 119 203 203 203 F F F
127802024-01-24T19:11:01.617Z 2 040-059 114 114 114 196 196 196 F F F
127812024-01-24T19:11:01.617Z 3 060-079 119 119 119 203 203 203 F F F
127822024-01-24T19:11:01.617Z 4 080-099 114 114 114 196 196 196 F F F
127832024-01-24T19:11:01.617Z 5 100-119 119 119 119 203 203 203 F F F
127842024-01-24T19:11:01.617Z 6 120-139 119 119 119 203 203 203 F F F
127852024-01-24T19:11:01.617Z 7 140-159 119 119 119 203 203 203 F F F
127862024-01-24T19:11:01.617Z 8 160-179 117 117 117 201 201 201 F F F
127872024-01-24T19:11:01.617Z 9 180-199 118 118 118 202 202 202 F F F
127882024-01-24T19:11:01.618Z 10 200-219 118 118 118 202 202 202 F F F
127892024-01-24T19:11:01.618Z 11 220-239 113 113 113 195 195 195 F F F
127902024-01-24T19:11:01.618Z 12 240-259 116 116 116 200 200 200 F F F
127912024-01-24T19:11:01.620Z 13 260-279 114 114 114 196 196 196 F F F
127922024-01-24T19:11:01.620Z 14 280-299 112 112 112 194 194 194 F F F
127932024-01-24T19:11:01.620Z 15 300-319 117 117 117 201 201 201 F F F
127942024-01-24T19:11:01.620Z 16 320-339 110 110 110 187 187 187 F F F
127952024-01-24T19:11:01.620Z 17 340-359 119 119 119 203 203 203 F F F
127962024-01-24T19:11:01.620Z 18 360-379 119 119 119 203 203 203 F F F
127972024-01-24T19:11:01.620Z 19 380-399 115 115 115 199 199 199 F F F
127982024-01-24T19:11:01.621Z 20 400-419 116 116 116 200 200 200 F F F
127992024-01-24T19:11:01.621Z 21 420-439 114 114 114 198 198 198 F F F
128002024-01-24T19:11:01.621Z 22 440-459 92 92 92 155 155 155 F F F
128012024-01-24T19:11:01.621Z 23 460-479 110 110 110 188 188 188 F F F
128022024-01-24T19:11:01.621Z 24 480-499 114 114 114 198 198 198 F F F
128032024-01-24T19:11:01.621Z 25 500-519 119 119 119 203 203 203 F F F
128042024-01-24T19:11:01.622Z 26 520-539 119 119 119 203 203 203 F F F
128052024-01-24T19:11:01.622Z 27 540-559 119 119 119 203 203 203 F F F
128062024-01-24T19:11:01.622Z 28 560-579 119 119 119 203 203 203 F F F
128072024-01-24T19:11:01.622Z 29 580-599 119 119 119 203 203 203 F F F
128082024-01-24T19:11:01.622ZMax gen: 119, Max flush: 203
128092024-01-24T19:11:01.622Z
128102024-01-24T19:11:01.622ZBegin loop 60
128112024-01-24T19:11:01.623ZDownstairs to restart: 2
128122024-01-24T19:11:01.635Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.301905376Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609}
128132024-01-24T19:11:01.635Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.302406955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609}
128142024-01-24T19:11:02.057Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.302486006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609}
128152024-01-24T19:11:02.057Z{"msg":"Crucible 7eec18dd-d574-4a50-b28f-3d492d954b88 has session id: 21213611-768b-4e1c-a1f6-5c13b2096e8f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30256615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128162024-01-24T19:11:02.057Z{"msg":"Crucible runtime is spawned
128172024-01-24T19:11:02.057ZUpstairs opts: Upstairs UUID: 7eec18dd-d574-4a50-b28f-3d492d954b88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30263608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128182024-01-24T19:11:02.057Z{"msg":"Crucible stats registered with UUID: 7eec18dd-d574-4a50-b28f-3d492d954b88","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.302717187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128192024-01-24T19:11:02.057Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.302750474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609}
128202024-01-24T19:11:02.057Z{"msg":"7eec18dd-d574-4a50-b28f-3d492d954b88 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.3031081Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128212024-01-24T19:11:02.057Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303164162Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128222024-01-24T19:11:02.057Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303191804Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128232024-01-24T19:11:02.059Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303219493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128242024-01-24T19:11:02.059Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303290574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128252024-01-24T19:11:02.060Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303365995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128262024-01-24T19:11:02.060Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30344568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128272024-01-24T19:11:02.060Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303488186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128282024-01-24T19:11:02.060Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303533242Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128292024-01-24T19:11:02.060Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303604631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128302024-01-24T19:11:02.060Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303695957Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128312024-01-24T19:11:02.060Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303773057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128322024-01-24T19:11:02.060Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.303900082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128332024-01-24T19:11:02.061Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.304986992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128342024-01-24T19:11:02.061Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.305054447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128352024-01-24T19:11:02.061Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.305260015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128362024-01-24T19:11:02.061Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.305360272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128372024-01-24T19:11:02.061Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.305995168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128382024-01-24T19:11:02.061Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30606892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128392024-01-24T19:11:02.061Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.306099632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128402024-01-24T19:11:02.061Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.306143382Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128412024-01-24T19:11:02.061Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307017191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128422024-01-24T19:11:02.061Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307079855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128432024-01-24T19:11:02.062Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307107941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128442024-01-24T19:11:02.062Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307230522Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128452024-01-24T19:11:02.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307297426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128462024-01-24T19:11:02.062Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307325031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128472024-01-24T19:11:02.062Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.307349003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128482024-01-24T19:11:02.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308118444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128492024-01-24T19:11:02.062Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308187075Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128502024-01-24T19:11:02.062Z{"msg":"[0]R flush_numbers[0..12]: [199, 203, 196, 203, 196, 203, 203, 203, 201, 202, 202, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308216982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128512024-01-24T19:11:02.062Z{"msg":"[0]R generation[0..12]: [115, 119, 114, 119, 114, 119, 119, 119, 117, 118, 118, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308240927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128522024-01-24T19:11:02.062Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308265105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128532024-01-24T19:11:02.063Z{"msg":"[1]R flush_numbers[0..12]: [199, 203, 196, 203, 196, 203, 203, 203, 201, 202, 202, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30828992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128542024-01-24T19:11:02.063Z{"msg":"[1]R generation[0..12]: [115, 119, 114, 119, 114, 119, 119, 119, 117, 118, 118, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308335497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128552024-01-24T19:11:02.063Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308362841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128562024-01-24T19:11:02.063Z{"msg":"[2]R flush_numbers[0..12]: [199, 203, 196, 203, 196, 203, 203, 203, 201, 202, 202, 195]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30840674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128572024-01-24T19:11:02.063Z{"msg":"[2]R generation[0..12]: [115, 119, 114, 119, 114, 119, 119, 119, 117, 118, 118, 113]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308433465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128582024-01-24T19:11:02.063Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
128592024-01-24T19:11:02.063Z,"time":"2024-01-24T19:11:00.308478432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128602024-01-24T19:11:02.063Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308508807Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128612024-01-24T19:11:02.063Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308551696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128622024-01-24T19:11:02.063Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308576461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128632024-01-24T19:11:02.064Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308619589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128642024-01-24T19:11:02.064Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308645295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128652024-01-24T19:11:02.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.3086658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128662024-01-24T19:11:02.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308717092Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128672024-01-24T19:11:02.064Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308743546Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128682024-01-24T19:11:02.064Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308811719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128692024-01-24T19:11:02.064Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308835952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128702024-01-24T19:11:02.064Z{"msg":"7eec18dd-d574-4a50-b28f-3d492d954b88 is now active with session: 21213611-768b-4e1c-a1f6-5c13b2096e8f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308863517Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128712024-01-24T19:11:02.064Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.308908437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609,"session_id":"21213611-768b-4e1c-a1f6-5c13b2096e8f"}
128722024-01-24T19:11:02.064Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:00.30893208Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1609}
128732024-01-24T19:11:02.064ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128742024-01-24T19:11:02.064ZLoading write count information from file "/tmp/test_repair_verify.data"
128752024-01-24T19:11:02.064ZRun Repair workload
128762024-01-24T19:11:02.064Z01/30 Write block 511 len 12288 data: 13 13 13
128772024-01-24T19:11:02.072Z02/30 Read block 555 len 4096
128782024-01-24T19:11:02.072Z03/30 Write block 437 len 16384 data: 10 7 8 8
128792024-01-24T19:11:02.073Z04/30 Read block 112 len 16384
128802024-01-24T19:11:02.073Z05/30 Read block 160 len 36864
128812024-01-24T19:11:02.073Z06/30 Read block 281 len 8192
128822024-01-24T19:11:02.073Z07/30 Read block 94 len 36864
128832024-01-24T19:11:02.073Z08/30 Write block 228 len 20480 data: 8 7 9 9 9
128842024-01-24T19:11:02.073Z09/30 Read block 224 len 16384
128852024-01-24T19:11:02.073Z10/30 Read block 41 len 40960
128862024-01-24T19:11:02.073Z11/30 Write block 360 len 32768 data: 10 15 14 13 15 14 13 12
128872024-01-24T19:11:02.073Z12/30 Read block 201 len 16384
128882024-01-24T19:11:02.073Z13/30 Write block 245 len 28672 data: 14 16 15 13 15 12 10
128892024-01-24T19:11:02.073Z14/30 Read block 63 len 4096
128902024-01-24T19:11:02.073Z15/30 Write block 367 len 4096 data: 13
128912024-01-24T19:11:02.073Z16/30 Write block 284 len 28672 data: 10 8 7 8 10 9 8
128922024-01-24T19:11:02.073Z17/30 Write block 517 len 20480 data: 15 15 14 12 11
128932024-01-24T19:11:02.073Z18/30 Write block 172 len 20480 data: 12 10 12 10 10
128942024-01-24T19:11:02.073Z19/30 Read block 470 len 16384
128952024-01-24T19:11:02.073Z20/30 Read block 165 len 28672
128962024-01-24T19:11:02.073Z21/30 Flush
128972024-01-24T19:11:02.073Z22/30 Write block 147 len 12288 data: 12 10 12
128982024-01-24T19:11:02.073Z23/30 Write block 60 len 24576 data: 9 9 8 10 12 9
128992024-01-24T19:11:02.073Z24/30 Write block 568 len 36864 data: 13 12 8 8 9 7 7 8 8
129002024-01-24T19:11:02.074Z25/30 Flush
129012024-01-24T19:11:02.075Z26/30 Write block 269 len 12288 data: 10 11 10
129022024-01-24T19:11:02.075Z27/30 Write block 158 len 16384 data: 12 8 7 15
129032024-01-24T19:11:02.075Z28/30 Read block 531 len 4096
129042024-01-24T19:11:02.075Z29/30 Read block 147 len 8192
129052024-01-24T19:11:02.075Z30/30 Read block 520 len 32768
129062024-01-24T19:11:02.075Z----------------------------------------------------------------
129072024-01-24T19:11:02.075Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129082024-01-24T19:11:02.075ZStates: Active Active Active
129092024-01-24T19:11:02.075ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129102024-01-24T19:11:02.075Z 1 Acked 1000 Write 3 Done Done Sent false
129112024-01-24T19:11:02.075Z 2 Acked 1001 Read 1 Done Done Sent false
129122024-01-24T19:11:02.075Z 3 Acked 1002 Write 4 Done Done Sent false
129132024-01-24T19:11:02.075Z 4 Acked 1003 Read 4 Done Done Sent false
129142024-01-24T19:11:02.075Z 5 Acked 1004 Read 9 Done Done Sent false
129152024-01-24T19:11:02.075Z 6 Acked 1005 Read 2 Done Done Sent false
129162024-01-24T19:11:02.075Z 7 Acked 1006 Read 9 Done Done Sent false
129172024-01-24T19:11:02.075Z 8 Acked 1007 Write 5 Done Done Sent false
129182024-01-24T19:11:02.075Z 9 Acked 1008 Read 4 Done Done Sent false
129192024-01-24T19:11:02.075Z 10 Acked 1009 Read 10 Done Done Sent false
129202024-01-24T19:11:02.075Z 11 Acked 1010 Write 8 Done Done Sent false
129212024-01-24T19:11:02.075Z 12 Acked 1011 Read 4 Done Done Sent false
129222024-01-24T19:11:02.075Z 13 Acked 1012 Write 7 Done Done Sent false
129232024-01-24T19:11:02.076Z 14 Acked 1013 Read 1 Done Done Sent false
129242024-01-24T19:11:02.076Z 15 Acked 1014 Write 1 Done Done Sent false
129252024-01-24T19:11:02.076Z 16 Acked 1015 Write 7 Done Done Sent false
129262024-01-24T19:11:02.076Z 17 Acked 1016 Write 5 Done Done Sent false
129272024-01-24T19:11:02.076Z 18 Acked 1017 Write 5 Done Done Sent false
129282024-01-24T19:11:02.076Z 19 Acked 1018 Read 4 Done Done Sent false
129292024-01-24T19:11:02.076Z 20 Acked 1019 Read 7 Done Done Sent false
129302024-01-24T19:11:02.077Z 21 Acked 1020 Flush 0 Done Done Sent false
129312024-01-24T19:11:02.077Z 22 Acked 1021 Write 3 Done Done Sent false
129322024-01-24T19:11:02.077Z 23 Acked 1022 Write 6 Done Done Sent false
129332024-01-24T19:11:02.077Z 24 Acked 1023 Write 9 Done Done Sent false
129342024-01-24T19:11:02.077Z 25 Acked 1024 Flush 0 Done Done Sent false
129352024-01-24T19:11:02.077Z 26 Acked 1025 Write 3 Done Done Sent false
129362024-01-24T19:11:02.077Z 27 Acked 1026 Write 4 Done Done Sent false
129372024-01-24T19:11:02.077Z 28 Acked 1027 Read 1 Done Done Sent false
129382024-01-24T19:11:02.077Z 29 Acked 1028 Read 2 Done Done Sent false
129392024-01-24T19:11:02.077Z 30 Acked 1029 Read 8 Done Done Sent false
129402024-01-24T19:11:02.077Z STATES DS:0 DS:1 DS:2 TOTAL
129412024-01-24T19:11:02.077Z New 0 0 0 0
129422024-01-24T19:11:02.077Z Sent 0 0 30 30
129432024-01-24T19:11:02.077Z Done 30 30 0 60
129442024-01-24T19:11:02.077Z Skipped 0 0 0 0
129452024-01-24T19:11:02.077Z Error 0 0 0 0
129462024-01-24T19:11:02.077ZLast Flush: 1024 1024 0
129472024-01-24T19:11:02.077ZDownstairs last five completed:
129482024-01-24T19:11:02.077ZUpstairs last five completed: 30 29 28 27 26
129492024-01-24T19:11:02.077ZWrote out file "/tmp/test_repair_verify.data"
129502024-01-24T19:11:02.077Z
129512024-01-24T19:11:02.078ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
129522024-01-24T19:11:02.078ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129532024-01-24T19:11:02.078Z 0 000-019 115 115 115 199 199 199 F F F
129542024-01-24T19:11:02.078Z 1 020-039 119 119 119 203 203 203 F F F
129552024-01-24T19:11:02.078Z 2 040-059 114 114 114 196 196 196 F F F
129562024-01-24T19:11:02.078Z 3 060-079 120 120 119 205 205 203 F F F <---
129572024-01-24T19:11:02.078Z 4 080-099 114 114 114 196 196 196 F F F
129582024-01-24T19:11:02.078Z 5 100-119 119 119 119 203 203 203 F F F
129592024-01-24T19:11:02.078Z 6 120-139 119 119 119 203 203 203 F F F
129602024-01-24T19:11:02.078Z 7 140-159 120 120 119 205 205 203 T T F <---
129612024-01-24T19:11:02.078Z 8 160-179 120 120 117 204 204 201 T T F <---
129622024-01-24T19:11:02.078Z 9 180-199 118 118 118 202 202 202 F F F
129632024-01-24T19:11:02.078Z 10 200-219 118 118 118 202 202 202 F F F
129642024-01-24T19:11:02.078Z 11 220-239 120 120 113 204 204 195 F F F <---
129652024-01-24T19:11:02.078Z 12 240-259 120 120 116 204 204 200 F F F <---
129662024-01-24T19:11:02.078Z 13 260-279 114 114 114 196 196 196 T T F <---
129672024-01-24T19:11:02.078Z 14 280-299 120 120 112 204 204 194 F F F <---
129682024-01-24T19:11:02.079Z 15 300-319 117 117 117 201 201 201 F F F
129692024-01-24T19:11:02.079Z 16 320-339 110 110 110 187 187 187 F F F
129702024-01-24T19:11:02.079Z 17 340-359 119 119 119 203 203 203 F F F
129712024-01-24T19:11:02.079Z 18 360-379 120 120 119 204 204 203 F F F <---
129722024-01-24T19:11:02.079Z 19 380-399 115 115 115 199 199 199 F F F
129732024-01-24T19:11:02.079Z 20 400-419 116 116 116 200 200 200 F F F
129742024-01-24T19:11:02.079Z 21 420-439 120 120 114 204 204 198 F F F <---
129752024-01-24T19:11:02.079Z 22 440-459 120 120 92 204 204 155 F F F <---
129762024-01-24T19:11:02.079Z 23 460-479 110 110 110 188 188 188 F F F
129772024-01-24T19:11:02.084Z 24 480-499 114 114 114 198 198 198 F F F
129782024-01-24T19:11:02.084Z 25 500-519 120 120 119 204 204 203 F F F <---
129792024-01-24T19:11:02.084Z 26 520-539 120 120 119 204 204 203 F F F <---
129802024-01-24T19:11:02.084Z 27 540-559 119 119 119 203 203 203 F F F
129812024-01-24T19:11:02.084Z 28 560-579 120 120 119 205 205 203 F F F <---
129822024-01-24T19:11:02.084Z 29 580-599 119 119 119 203 203 203 F F F
129832024-01-24T19:11:02.085ZMax gen: 120, Max flush: 205
129842024-01-24T19:11:02.085ZError: Difference in extent metadata found!
129852024-01-24T19:11:02.085ZA Difference in extent metadata is expected here
129862024-01-24T19:11:02.085ZOn loop 60
129872024-01-24T19:11:02.085Z
129882024-01-24T19:11:02.085ZVerifying data now
129892024-01-24T19:11:06.884ZNo change after verify
129902024-01-24T19:11:06.884ZLoop: 60 Downstairs dump after verify (and repair):
129912024-01-24T19:11:06.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129922024-01-24T19:11:06.913Z 0 000-019 115 115 115 199 199 199 F F F
129932024-01-24T19:11:06.913Z 1 020-039 119 119 119 203 203 203 F F F
129942024-01-24T19:11:06.913Z 2 040-059 114 114 114 196 196 196 F F F
129952024-01-24T19:11:06.913Z 3 060-079 120 120 120 205 205 205 F F F
129962024-01-24T19:11:06.913Z 4 080-099 114 114 114 196 196 196 F F F
129972024-01-24T19:11:06.913Z 5 100-119 119 119 119 203 203 203 F F F
129982024-01-24T19:11:06.913Z 6 120-139 119 119 119 203 203 203 F F F
129992024-01-24T19:11:06.913Z 7 140-159 121 121 121 206 206 206 F F F
130002024-01-24T19:11:06.913Z 8 160-179 121 121 121 206 206 206 F F F
130012024-01-24T19:11:06.924Z 9 180-199 118 118 118 202 202 202 F F F
130022024-01-24T19:11:06.924Z 10 200-219 118 118 118 202 202 202 F F F
130032024-01-24T19:11:06.924Z 11 220-239 120 120 120 204 204 204 F F F
130042024-01-24T19:11:06.924Z 12 240-259 120 120 120 204 204 204 F F F
130052024-01-24T19:11:06.924Z 13 260-279 121 121 121 206 206 206 F F F
130062024-01-24T19:11:06.924Z 14 280-299 120 120 120 204 204 204 F F F
130072024-01-24T19:11:06.925Z 15 300-319 117 117 117 201 201 201 F F F
130082024-01-24T19:11:06.925Z 16 320-339 110 110 110 187 187 187 F F F
130092024-01-24T19:11:06.925Z 17 340-359 119 119 119 203 203 203 F F F
130102024-01-24T19:11:06.925Z 18 360-379 120 120 120 204 204 204 F F F
130112024-01-24T19:11:06.925Z 19 380-399 115 115 115 199 199 199 F F F
130122024-01-24T19:11:06.925Z 20 400-419 116 116 116 200 200 200 F F F
130132024-01-24T19:11:06.925Z 21 420-439 120 120 120 204 204 204 F F F
130142024-01-24T19:11:06.925Z 22 440-459 120 120 120 204 204 204 F F F
130152024-01-24T19:11:06.925Z 23 460-479 110 110 110 188 188 188 F F F
130162024-01-24T19:11:06.925Z 24 480-499 114 114 114 198 198 198 F F F
130172024-01-24T19:11:06.925Z 25 500-519 120 120 120 204 204 204 F F F
130182024-01-24T19:11:06.925Z 26 520-539 120 120 120 204 204 204 F F F
130192024-01-24T19:11:06.925Z 27 540-559 119 119 119 203 203 203 F F F
130202024-01-24T19:11:06.925Z 28 560-579 120 120 120 205 205 205 F F F
130212024-01-24T19:11:06.925Z 29 580-599 119 119 119 203 203 203 F F F
130222024-01-24T19:11:06.925ZMax gen: 121, Max flush: 206
130232024-01-24T19:11:06.925Z
130242024-01-24T19:11:06.925ZBegin loop 61
130252024-01-24T19:11:06.925ZDownstairs to restart: 2
130262024-01-24T19:11:06.944Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.612193309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617}
130272024-01-24T19:11:06.944Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.612644443Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617}
130282024-01-24T19:11:06.959Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.61272494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617}
130292024-01-24T19:11:06.959Z{"msg":"Crucible runtime is spawned
130302024-01-24T19:11:06.960ZCrucible 35d5594c-62a9-426d-b69d-2dbdf09d8c5e has session id: 8474efa6-5f97-4160-9aef-6acf16f77ea7","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.612999218Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130312024-01-24T19:11:06.960Z{"msg":"Upstairs opts: Upstairs UUID: 35d5594c-62a9-426d-b69d-2dbdf09d8c5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613089687Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130322024-01-24T19:11:06.960Z{"msg":"Crucible stats registered with UUID: 35d5594c-62a9-426d-b69d-2dbdf09d8c5e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613214551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130332024-01-24T19:11:06.960Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613257474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617}
130342024-01-24T19:11:06.960Z{"msg":"35d5594c-62a9-426d-b69d-2dbdf09d8c5e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613311938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130352024-01-24T19:11:06.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613352337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130362024-01-24T19:11:06.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613426025Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130372024-01-24T19:11:06.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613521894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130382024-01-24T19:11:06.960Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613568107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130392024-01-24T19:11:06.960Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.61361103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130402024-01-24T19:11:06.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613723332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130412024-01-24T19:11:06.961Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.613957693Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130422024-01-24T19:11:06.961Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.614071105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130432024-01-24T19:11:06.961Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.614160068Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130442024-01-24T19:11:06.961Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.614190703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130452024-01-24T19:11:06.962Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.614249949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130462024-01-24T19:11:06.962Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.61429254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130472024-01-24T19:11:06.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.615181235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130482024-01-24T19:11:06.962Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.615259838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130492024-01-24T19:11:06.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.615406146Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130502024-01-24T19:11:06.962Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.615494287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130512024-01-24T19:11:06.962Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.616137859Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130522024-01-24T19:11:06.963Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.616415861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130532024-01-24T19:11:06.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.616977715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130542024-01-24T19:11:06.963Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617036518Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130552024-01-24T19:11:06.963Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617153454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130562024-01-24T19:11:06.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617195817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130572024-01-24T19:11:06.963Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617251888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130582024-01-24T19:11:06.963Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617295129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130592024-01-24T19:11:06.963Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617348638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130602024-01-24T19:11:06.963Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.617392003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130612024-01-24T19:11:06.963Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618040333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130622024-01-24T19:11:06.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618724427Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130632024-01-24T19:11:06.963Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618770489Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130642024-01-24T19:11:06.963Z{"msg":"[0]R flush_numbers[0..12]: [199, 203, 196, 205, 196, 203, 203, 206, 206, 202, 202, 204]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618799652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130652024-01-24T19:11:06.963Z{"msg":"[0]R generation[0..12]: [115, 119, 114, 120, 114, 119, 119, 121, 121, 118, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618880642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130662024-01-24T19:11:06.963Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.61891046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130672024-01-24T19:11:06.963Z{"msg":"[1]R flush_numbers[0..12]: [199, 203, 196, 205, 196, 203, 203, 206, 206, 202, 202, 204]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.618971528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130682024-01-24T19:11:06.963Z{"msg":"[1]R generation[0..12]: [115, 119, 114, 120, 114, 119, 119, 121, 121, 118, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619027479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
130692024-01-24T19:11:06.963Z,"":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130702024-01-24T19:11:06.963Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619156437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130712024-01-24T19:11:06.963Z{"msg":"[2]R flush_numbers[0..12]: [199, 203, 196, 205, 196, 203, 203, 206, 206, 202, 202, 204]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619197457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130722024-01-24T19:11:06.963Z{"msg":"[2]R generation[0..12]: [115, 119, 114, 120, 114, 119, 119, 121, 121, 118, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619253519Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130732024-01-24T19:11:06.963Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619282009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130742024-01-24T19:11:06.964Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619341483Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130752024-01-24T19:11:06.964Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619371486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130762024-01-24T19:11:06.964Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619430713Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130772024-01-24T19:11:06.964Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619462487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130782024-01-24T19:11:06.964Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619520238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130792024-01-24T19:11:06.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619545255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130802024-01-24T19:11:06.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619604453Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130812024-01-24T19:11:06.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619636692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130822024-01-24T19:11:06.965Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619702303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130832024-01-24T19:11:06.965Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619733434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130842024-01-24T19:11:06.965Z{"msg":"35d5594c-62a9-426d-b69d-2dbdf09d8c5e is now active with session: 8474efa6-5f97-4160-9aef-6acf16f77ea7","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619802038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130852024-01-24T19:11:06.965Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:05.619831773Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617,"session_id":"8474efa6-5f97-4160-9aef-6acf16f77ea7"}
130862024-01-24T19:11:06.965Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
130872024-01-24T19:11:06.965Z,"time":"2024-01-24T19:11:05.619891059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1617}
130882024-01-24T19:11:06.965ZLoading write count information from file "/tmp/test_repair_verify.data"
130892024-01-24T19:11:06.965ZRun Repair workload
130902024-01-24T19:11:07.008Z01/30 Write block 272 len 20480 data: 12 13 11 11 9
130912024-01-24T19:11:07.008Z02/30 Read block 154 len 12288
130922024-01-24T19:11:07.008Z03/30 Write block 497 len 40960 data: 13 14 13 10 10 9 6 9 9 8
130932024-01-24T19:11:07.008Z04/30 Flush
130942024-01-24T19:11:07.008Z05/30 Write block 233 len 8192 data: 11 12
130952024-01-24T19:11:07.008Z06/30 Write block 314 len 12288 data: 12 13 14
130962024-01-24T19:11:07.008Z07/30 Read block 408 len 12288
130972024-01-24T19:11:07.008Z08/30 Write block 142 len 20480 data: 5 5 7 8 10
130982024-01-24T19:11:07.008Z09/30 Write block 174 len 20480 data: 13 11 11 9 10
130992024-01-24T19:11:07.009Z10/30 Read block 332 len 40960
131002024-01-24T19:11:07.009Z11/30 Read block 325 len 24576
131012024-01-24T19:11:07.009Z12/30 Read block 282 len 20480
131022024-01-24T19:11:07.018Z13/30 Write block 346 len 8192 data: 7 7
131032024-01-24T19:11:07.019Z14/30 Read block 75 len 36864
131042024-01-24T19:11:07.019Z15/30 Write block 27 len 12288 data: 12 14 14
131052024-01-24T19:11:07.019Z16/30 Flush
131062024-01-24T19:11:07.019Z17/30 Write block 327 len 36864 data: 9 11 12 10 10 12 10 9 10
131072024-01-24T19:11:07.019Z18/30 Write block 388 len 16384 data: 7 7 7 8
131082024-01-24T19:11:07.019Z19/30 Read block 137 len 4096
131092024-01-24T19:11:07.019Z20/30 Write block 514 len 28672 data: 12 13 11 16 16 15 13
131102024-01-24T19:11:07.019Z21/30 Read block 436 len 24576
131112024-01-24T19:11:07.019Z22/30 Write block 379 len 20480 data: 8 11 10 8 9
131122024-01-24T19:11:07.019Z23/30 Read block 269 len 8192
131132024-01-24T19:11:07.019Z24/30 Read block 580 len 32768
131142024-01-24T19:11:07.019Z25/30 Write block 584 len 4096 data: 9
131152024-01-24T19:11:07.019Z26/30 Write block 594 len 20480 data: 10 7 8 5 5
131162024-01-24T19:11:07.019Z27/30 Write block 435 len 20480 data: 9 8 11 8 9
131172024-01-24T19:11:07.019Z28/30 Write block 468 len 40960 data: 9 7 8 10 14 11 12 13 12 11
131182024-01-24T19:11:07.020Z29/30 Read block 557 len 36864
131192024-01-24T19:11:07.020Z30/30 Read block 312 len 36864
131202024-01-24T19:11:07.020Z----------------------------------------------------------------
131212024-01-24T19:11:07.020Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
131222024-01-24T19:11:07.020ZStates: Active Active Active
131232024-01-24T19:11:07.020ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131242024-01-24T19:11:07.020Z 1 Acked 1000 Write 5 Done Done Sent false
131252024-01-24T19:11:07.020Z 2 Acked 1001 Read 3 Done Done Sent false
131262024-01-24T19:11:07.020Z 3 Acked 1002 Write 10 Done Done Sent false
131272024-01-24T19:11:07.020Z 4 Acked 1003 Flush 0 Done Done Sent false
131282024-01-24T19:11:07.020Z 5 Acked 1004 Write 2 Done Done Sent false
131292024-01-24T19:11:07.020Z 6 Acked 1005 Write 3 Done Done Sent false
131302024-01-24T19:11:07.020Z 7 Acked 1006 Read 3 Done Done Sent false
131312024-01-24T19:11:07.020Z 8 Acked 1007 Write 5 Done Done Sent false
131322024-01-24T19:11:07.020Z 9 Acked 1008 Write 5 Done Done Sent false
131332024-01-24T19:11:07.020Z 10 Acked 1009 Read 10 Done Done Sent false
131342024-01-24T19:11:07.020Z 11 Acked 1010 Read 6 Done Done Sent false
131352024-01-24T19:11:07.020Z 12 Acked 1011 Read 5 Done Done Sent false
131362024-01-24T19:11:07.020Z 13 Acked 1012 Write 2 Done Done Sent false
131372024-01-24T19:11:07.020Z 14 Acked 1013 Read 9 Done Done Sent false
131382024-01-24T19:11:07.020Z 15 Acked 1014 Write 3 Done Done Sent false
131392024-01-24T19:11:07.021Z 16 Acked 1015 Flush 0 Done Done Sent false
131402024-01-24T19:11:07.021Z 17 Acked 1016 Write 9 Done Done Sent false
131412024-01-24T19:11:07.023Z 18 Acked 1017 Write 4 Done Done Sent false
131422024-01-24T19:11:07.023Z 19 Acked 1018 Read 1 Done Done Sent false
131432024-01-24T19:11:07.023Z 20 Acked 1019 Write 7 Done Done Sent false
131442024-01-24T19:11:07.024Z 21 Acked 1020 Read 6 Done Done Sent false
131452024-01-24T19:11:07.024Z 22 Acked 1021 Write 5 Done Done Sent false
131462024-01-24T19:11:07.024Z 23 Acked 1022 Read 2 Done Done Sent false
131472024-01-24T19:11:07.024Z 24 Acked 1023 Read 8 Done Done Sent false
131482024-01-24T19:11:07.024Z 25 Acked 1024 Write 1 Done Done Sent false
131492024-01-24T19:11:07.024Z 26 Acked 1025 Write 5 Done Done Sent false
131502024-01-24T19:11:07.024Z 27 Acked 1026 Write 5 Done Done Sent false
131512024-01-24T19:11:07.024Z 28 Acked 1027 Write 10 Done Done Sent false
131522024-01-24T19:11:07.024Z 29 Acked 1028 Read 9 Done Done Sent false
131532024-01-24T19:11:07.025Z 30 Acked 1029 Read 9 Sent Done Sent false
131542024-01-24T19:11:07.025Z STATES DS:0 DS:1 DS:2 TOTAL
131552024-01-24T19:11:07.025Z New 0 0 0 0
131562024-01-24T19:11:07.025Z Sent 1 0 30 31
131572024-01-24T19:11:07.025Z Done 29 30 0 59
131582024-01-24T19:11:07.025Z Skipped 0 0 0 0
131592024-01-24T19:11:07.025Z Error 0 0 0 0
131602024-01-24T19:11:07.025ZLast Flush: 1015 1015 0
131612024-01-24T19:11:07.025ZDownstairs last five completed:
131622024-01-24T19:11:07.025ZUpstairs last five completed: 30 29 28 27 26
131632024-01-24T19:11:07.025ZWrote out file "/tmp/test_repair_verify.data"
131642024-01-24T19:11:07.025Z
131652024-01-24T19:11:07.025ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131662024-01-24T19:11:07.041ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131672024-01-24T19:11:07.041Z 0 000-019 115 115 115 199 199 199 F F F
131682024-01-24T19:11:07.055Z 1 020-039 122 122 119 208 208 203 F F F <---
131692024-01-24T19:11:07.055Z 2 040-059 114 114 114 196 196 196 F F F
131702024-01-24T19:11:07.055Z 3 060-079 120 120 120 205 205 205 F F F
131712024-01-24T19:11:07.055Z 4 080-099 114 114 114 196 196 196 F F F
131722024-01-24T19:11:07.055Z 5 100-119 119 119 119 203 203 203 F F F
131732024-01-24T19:11:07.055Z 6 120-139 119 119 119 203 203 203 F F F
131742024-01-24T19:11:07.055Z 7 140-159 122 122 121 208 208 206 F F F <---
131752024-01-24T19:11:07.057Z 8 160-179 122 122 121 208 208 206 F F F <---
131762024-01-24T19:11:07.057Z 9 180-199 118 118 118 202 202 202 F F F
131772024-01-24T19:11:07.057Z 10 200-219 118 118 118 202 202 202 F F F
131782024-01-24T19:11:07.057Z 11 220-239 122 122 120 208 208 204 F F F <---
131792024-01-24T19:11:07.057Z 12 240-259 120 120 120 204 204 204 F F F
131802024-01-24T19:11:07.057Z 13 260-279 122 122 121 207 207 206 F F F <---
131812024-01-24T19:11:07.058Z 14 280-299 120 120 120 204 204 204 F F F
131822024-01-24T19:11:07.058Z 15 300-319 122 122 117 208 208 201 F F F <---
131832024-01-24T19:11:07.058Z 16 320-339 110 110 110 187 187 187 T T F <---
131842024-01-24T19:11:07.058Z 17 340-359 122 122 119 208 208 203 F F F <---
131852024-01-24T19:11:07.058Z 18 360-379 120 120 120 204 204 204 T T F <---
131862024-01-24T19:11:07.058Z 19 380-399 115 115 115 199 199 199 T T F <---
131872024-01-24T19:11:07.058Z 20 400-419 116 116 116 200 200 200 F F F
131882024-01-24T19:11:07.058Z 21 420-439 120 120 120 204 204 204 T T F <---
131892024-01-24T19:11:07.058Z 22 440-459 120 120 120 204 204 204 F F F
131902024-01-24T19:11:07.059Z 23 460-479 110 110 110 188 188 188 T T F <---
131912024-01-24T19:11:07.059Z 24 480-499 122 122 114 207 207 198 F F F <---
131922024-01-24T19:11:07.059Z 25 500-519 122 122 120 207 207 204 T T F <---
131932024-01-24T19:11:07.059Z 26 520-539 120 120 120 204 204 204 T T F <---
131942024-01-24T19:11:07.059Z 27 540-559 119 119 119 203 203 203 F F F
131952024-01-24T19:11:07.060Z 28 560-579 120 120 120 205 205 205 F F F
131962024-01-24T19:11:07.060Z 29 580-599 119 119 119 203 203 203 T T F <---
131972024-01-24T19:11:07.060ZMax gen: 122, Max flush: 208
131982024-01-24T19:11:07.060ZError: Difference in extent metadata found!
131992024-01-24T19:11:07.060ZA Difference in extent metadata is expected here
132002024-01-24T19:11:07.060ZOn loop 61
132012024-01-24T19:11:07.060Z
132022024-01-24T19:11:07.070ZVerifying data now
132032024-01-24T19:11:12.425ZNo change after verify
132042024-01-24T19:11:12.425ZLoop: 61 Downstairs dump after verify (and repair):
132052024-01-24T19:11:12.446ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132062024-01-24T19:11:12.446Z 0 000-019 115 115 115 199 199 199 F F F
132072024-01-24T19:11:12.451Z 1 020-039 122 122 122 208 208 208 F F F
132082024-01-24T19:11:12.451Z 2 040-059 114 114 114 196 196 196 F F F
132092024-01-24T19:11:12.451Z 3 060-079 120 120 120 205 205 205 F F F
132102024-01-24T19:11:12.451Z 4 080-099 114 114 114 196 196 196 F F F
132112024-01-24T19:11:12.451Z 5 100-119 119 119 119 203 203 203 F F F
132122024-01-24T19:11:12.451Z 6 120-139 119 119 119 203 203 203 F F F
132132024-01-24T19:11:12.451Z 7 140-159 122 122 122 208 208 208 F F F
132142024-01-24T19:11:12.452Z 8 160-179 122 122 122 208 208 208 F F F
132152024-01-24T19:11:12.452Z 9 180-199 118 118 118 202 202 202 F F F
132162024-01-24T19:11:12.452Z 10 200-219 118 118 118 202 202 202 F F F
132172024-01-24T19:11:12.452Z 11 220-239 122 122 122 208 208 208 F F F
132182024-01-24T19:11:12.452Z 12 240-259 120 120 120 204 204 204 F F F
132192024-01-24T19:11:12.452Z 13 260-279 122 122 122 207 207 207 F F F
132202024-01-24T19:11:12.452Z 14 280-299 120 120 120 204 204 204 F F F
132212024-01-24T19:11:12.453Z 15 300-319 122 122 122 208 208 208 F F F
132222024-01-24T19:11:12.453Z 16 320-339 123 123 123 209 209 209 F F F
132232024-01-24T19:11:12.453Z 17 340-359 122 122 122 208 208 208 F F F
132242024-01-24T19:11:12.453Z 18 360-379 123 123 123 209 209 209 F F F
132252024-01-24T19:11:12.453Z 19 380-399 123 123 123 209 209 209 F F F
132262024-01-24T19:11:12.455Z 20 400-419 116 116 116 200 200 200 F F F
132272024-01-24T19:11:12.455Z 21 420-439 123 123 123 209 209 209 F F F
132282024-01-24T19:11:12.455Z 22 440-459 120 120 120 204 204 204 F F F
132292024-01-24T19:11:12.456Z 23 460-479 123 123 123 209 209 209 F F F
132302024-01-24T19:11:12.456Z 24 480-499 122 122 122 207 207 207 F F F
132312024-01-24T19:11:12.456Z 25 500-519 123 123 123 209 209 209 F F F
132322024-01-24T19:11:12.456Z 26 520-539 123 123 123 209 209 209 F F F
132332024-01-24T19:11:12.456Z 27 540-559 119 119 119 203 203 203 F F F
132342024-01-24T19:11:12.456Z 28 560-579 120 120 120 205 205 205 F F F
132352024-01-24T19:11:12.457Z 29 580-599 123 123 123 209 209 209 F F F
132362024-01-24T19:11:12.457ZMax gen: 123, Max flush: 209
132372024-01-24T19:11:12.465Z
132382024-01-24T19:11:12.465ZBegin loop 62
132392024-01-24T19:11:12.465ZDownstairs to restart: 2
132402024-01-24T19:11:12.484Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.152835141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625}
132412024-01-24T19:11:12.484Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153263603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625}
132422024-01-24T19:11:12.484Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153297077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625}
132432024-01-24T19:11:12.493Z{"msg":"Crucible 56809081-46b0-4a40-8a70-dabb14a552ec has session id: a3601e4b-0888-41d7-950d-53041551118d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153349229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132442024-01-24T19:11:12.493Z{"msg":"Upstairs opts: Upstairs UUID: 56809081-46b0-4a40-8a70-dabb14a552ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153427886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132452024-01-24T19:11:12.494Z{"Crucible runtime is spawned
132462024-01-24T19:11:12.494Zmsg":"Crucible stats registered with UUID: 56809081-46b0-4a40-8a70-dabb14a552ec","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.15369013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132472024-01-24T19:11:12.494Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153775094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625}
132482024-01-24T19:11:12.494Z{"msg":"56809081-46b0-4a40-8a70-dabb14a552ec active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153868067Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132492024-01-24T19:11:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.15391096Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132502024-01-24T19:11:12.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.153953228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132512024-01-24T19:11:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154026843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132522024-01-24T19:11:12.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154067756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132532024-01-24T19:11:12.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154136079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132542024-01-24T19:11:12.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154176709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132552024-01-24T19:11:12.495Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154214954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132562024-01-24T19:11:12.495Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154322942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132572024-01-24T19:11:12.495Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154396406Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132582024-01-24T19:11:12.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154552967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132592024-01-24T19:11:12.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154595606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132602024-01-24T19:11:12.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.154635574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132612024-01-24T19:11:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.155851797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132622024-01-24T19:11:12.495Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.155979065Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132632024-01-24T19:11:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.156103929Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132642024-01-24T19:11:12.495Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.156152963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132652024-01-24T19:11:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.156944262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132662024-01-24T19:11:12.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.157267249Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132672024-01-24T19:11:12.495Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.157336973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132682024-01-24T19:11:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.157380572Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132692024-01-24T19:11:12.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.157963339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132702024-01-24T19:11:12.495Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.158032965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132712024-01-24T19:11:12.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.158078203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132722024-01-24T19:11:12.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.158409161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132732024-01-24T19:11:12.495Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.158473156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132742024-01-24T19:11:12.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.158500605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132752024-01-24T19:11:12.497Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.15852473Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132762024-01-24T19:11:12.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.15929331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132772024-01-24T19:11:12.508Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159383357Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132782024-01-24T19:11:12.508Z{"msg":"[0]R flush_numbers[0..12]: [199, 208, 196, 205, 196, 203, 203, 208, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159430484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132792024-01-24T19:11:12.508Z{"msg":"[0]R generation[0..12]: [115, 122, 114, 120, 114, 119, 119, 122, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159504211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132802024-01-24T19:11:12.508Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159580133Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132812024-01-24T19:11:12.508Z{"msg":"[1]R flush_numbers[0..12]: [199, 208, 196, 205, 196, 203, 203, 208, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159655402Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132822024-01-24T19:11:12.508Z{"msg":"Wait for a query_work_queue command to finish before sending IO
132832024-01-24T19:11:12.508Z[1]R generation[0..12]: [115, 122, 114, 120, 114, 119, 119, 122, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159820826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132842024-01-24T19:11:12.508Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159901956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132852024-01-24T19:11:12.511Z{"msg":"[2]R flush_numbers[0..12]: [199, 208, 196, 205, 196, 203, 203, 208, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.159980786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132862024-01-24T19:11:12.511Z{"msg":"[2]R generation[0..12]: [115, 122, 114, 120, 114, 119, 119, 122, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160069345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132872024-01-24T19:11:12.511Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160134598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132882024-01-24T19:11:12.511Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.16016627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132892024-01-24T19:11:12.511Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160257639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132902024-01-24T19:11:12.511Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160328782Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132912024-01-24T19:11:12.511Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160407652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132922024-01-24T19:11:12.511Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160486637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132932024-01-24T19:11:12.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160557955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132942024-01-24T19:11:12.511ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132952024-01-24T19:11:12.513Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160634719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132962024-01-24T19:11:12.513Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160702561Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132972024-01-24T19:11:12.513Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160732128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132982024-01-24T19:11:12.513Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160774401Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
132992024-01-24T19:11:12.513Z{"msg":"56809081-46b0-4a40-8a70-dabb14a552ec is now active with session: a3601e4b-0888-41d7-950d-53041551118d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160799398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
133002024-01-24T19:11:12.513Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160840877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625,"session_id":"a3601e4b-0888-41d7-950d-53041551118d"}
133012024-01-24T19:11:12.513Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:11.160864151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1625}
133022024-01-24T19:11:12.513ZLoading write count information from file "/tmp/test_repair_verify.data"
133032024-01-24T19:11:12.513ZRun Repair workload
133042024-01-24T19:11:12.513Z01/30 Write block 119 len 4096 data: 10
133052024-01-24T19:11:12.513Z02/30 Read block 270 len 12288
133062024-01-24T19:11:12.513Z03/30 Flush
133072024-01-24T19:11:12.513Z04/30 Write block 430 len 36864 data: 12 12 14 14 14 10 9 12 9
133082024-01-24T19:11:12.513Z05/30 Read block 300 len 16384
133092024-01-24T19:11:12.513Z06/30 Read block 576 len 12288
133102024-01-24T19:11:12.513Z07/30 Write block 285 len 8192 data: 9 8
133112024-01-24T19:11:12.513Z08/30 Flush
133122024-01-24T19:11:12.521Z09/30 Write block 501 len 36864 data: 11 10 7 10 10 9 10 9 9
133132024-01-24T19:11:12.521Z10/30 Read block 388 len 28672
133142024-01-24T19:11:12.521Z11/30 Read block 262 len 16384
133152024-01-24T19:11:12.521Z12/30 Read block 321 len 32768
133162024-01-24T19:11:12.521Z13/30 Write block 336 len 4096 data: 10
133172024-01-24T19:11:12.521Z14/30 Write block 148 len 4096 data: 11
133182024-01-24T19:11:12.521Z15/30 Read block 565 len 32768
133192024-01-24T19:11:12.521Z16/30 Read block 389 len 32768
133202024-01-24T19:11:12.521Z17/30 Read block 21 len 24576
133212024-01-24T19:11:12.521Z18/30 Flush
133222024-01-24T19:11:12.526Z19/30 Write block 66 len 8192 data: 6 5
133232024-01-24T19:11:12.526Z20/30 Read block 191 len 32768
133242024-01-24T19:11:12.526Z21/30 Read block 135 len 40960
133252024-01-24T19:11:12.526Z22/30 Read block 460 len 4096
133262024-01-24T19:11:12.530Z23/30 Read block 530 len 28672
133272024-01-24T19:11:12.530Z24/30 Read block 263 len 36864
133282024-01-24T19:11:12.537Z25/30 Read block 590 len 16384
133292024-01-24T19:11:12.537Z26/30 Write block 363 len 4096 data: 14
133302024-01-24T19:11:12.537Z27/30 Read block 383 len 16384
133312024-01-24T19:11:12.537Z28/30 Read block 270 len 8192
133322024-01-24T19:11:12.537Z29/30 Read block 316 len 8192
133332024-01-24T19:11:12.542Z30/30 Write block 425 len 36864 data: 5 5 9 10 12 13 13 15 15
133342024-01-24T19:11:12.542Z----------------------------------------------------------------
133352024-01-24T19:11:12.542Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
133362024-01-24T19:11:12.542ZStates: Active Active Active
133372024-01-24T19:11:12.542ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133382024-01-24T19:11:12.542Z 1 Acked 1000 Write 1 Done Done Sent false
133392024-01-24T19:11:12.542Z 2 Acked 1001 Read 3 Done Done Sent false
133402024-01-24T19:11:12.543Z 3 Acked 1002 Flush 0 Done Done Sent false
133412024-01-24T19:11:12.543Z 4 Acked 1003 Write 9 Done Done Sent false
133422024-01-24T19:11:12.543Z 5 Acked 1004 Read 4 Done Done Sent false
133432024-01-24T19:11:12.543Z 6 Acked 1005 Read 3 Done Done Sent false
133442024-01-24T19:11:12.543Z 7 Acked 1006 Write 2 Done Done Sent false
133452024-01-24T19:11:12.543Z 8 Acked 1007 Flush 0 Done Done Sent false
133462024-01-24T19:11:12.543Z 9 Acked 1008 Write 9 Done Done Sent false
133472024-01-24T19:11:12.544Z 10 Acked 1009 Read 7 Done Done Sent false
133482024-01-24T19:11:12.544Z 11 Acked 1010 Read 4 Done Done Sent false
133492024-01-24T19:11:12.544Z 12 Acked 1011 Read 8 Done Done Sent false
133502024-01-24T19:11:12.545Z 13 Acked 1012 Write 1 Done Done Sent false
133512024-01-24T19:11:12.545Z 14 Acked 1013 Write 1 Done Done Sent false
133522024-01-24T19:11:12.545Z 15 Acked 1014 Read 8 Done Done Sent false
133532024-01-24T19:11:12.545Z 16 Acked 1015 Read 8 Done Done Sent false
133542024-01-24T19:11:12.545Z 17 Acked 1016 Read 6 Done Done Sent false
133552024-01-24T19:11:12.546Z 18 Acked 1017 Flush 0 Done Done Sent false
133562024-01-24T19:11:12.546Z 19 Acked 1018 Write 2 Done Done Sent false
133572024-01-24T19:11:12.546Z 20 Acked 1019 Read 8 Done Done Sent false
133582024-01-24T19:11:12.546Z 21 Acked 1020 Read 10 Done Done Sent false
133592024-01-24T19:11:12.546Z 22 Acked 1021 Read 1 Done Done Sent false
133602024-01-24T19:11:12.547Z 23 Acked 1022 Read 7 Done Done Sent false
133612024-01-24T19:11:12.547Z 24 Acked 1023 Read 9 Done Done Sent false
133622024-01-24T19:11:12.547Z 25 Acked 1024 Read 4 Done Done Sent false
133632024-01-24T19:11:12.547Z 26 Acked 1025 Write 1 Done Done Sent false
133642024-01-24T19:11:12.547Z 27 Acked 1026 Read 4 Done Done Sent false
133652024-01-24T19:11:12.548Z 28 Acked 1027 Read 2 Done Done Sent false
133662024-01-24T19:11:12.548Z 29 Acked 1028 Read 2 Done Done Sent false
133672024-01-24T19:11:12.548Z 30 Acked 1029 Write 9 Sent Sent Sent false
133682024-01-24T19:11:12.548Z STATES DS:0 DS:1 DS:2 TOTAL
133692024-01-24T19:11:12.549Z New 0 0 0 0
133702024-01-24T19:11:12.549Z Sent 1 1 30 32
133712024-01-24T19:11:12.549Z Done 29 29 0 58
133722024-01-24T19:11:12.549Z Skipped 0 0 0 0
133732024-01-24T19:11:12.550Z Error 0 0 0 0
133742024-01-24T19:11:12.550ZLast Flush: 1017 1017 0
133752024-01-24T19:11:12.550ZDownstairs last five completed:
133762024-01-24T19:11:12.550ZUpstairs last five completed: 30 29 28 27 26
133772024-01-24T19:11:12.550ZWrote out file "/tmp/test_repair_verify.data"
133782024-01-24T19:11:12.557Z
133792024-01-24T19:11:12.558ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133802024-01-24T19:11:12.577ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133812024-01-24T19:11:12.577Z 0 000-019 115 115 115 199 199 199 F F F
133822024-01-24T19:11:12.577Z 1 020-039 122 122 122 208 208 208 F F F
133832024-01-24T19:11:12.577Z 2 040-059 114 114 114 196 196 196 F F F
133842024-01-24T19:11:12.577Z 3 060-079 120 120 120 205 205 205 T T F <---
133852024-01-24T19:11:12.578Z 4 080-099 114 114 114 196 196 196 F F F
133862024-01-24T19:11:12.578Z 5 100-119 124 124 119 210 210 203 F F F <---
133872024-01-24T19:11:12.582Z 6 120-139 119 119 119 203 203 203 F F F
133882024-01-24T19:11:12.582Z 7 140-159 124 124 122 212 212 208 F F F <---
133892024-01-24T19:11:12.582Z 8 160-179 122 122 122 208 208 208 F F F
133902024-01-24T19:11:12.582Z 9 180-199 118 118 118 202 202 202 F F F
133912024-01-24T19:11:12.582Z 10 200-219 118 118 118 202 202 202 F F F
133922024-01-24T19:11:12.582Z 11 220-239 122 122 122 208 208 208 F F F
133932024-01-24T19:11:12.582Z 12 240-259 120 120 120 204 204 204 F F F
133942024-01-24T19:11:12.582Z 13 260-279 122 122 122 207 207 207 F F F
133952024-01-24T19:11:12.582Z 14 280-299 124 124 120 211 211 204 F F F <---
133962024-01-24T19:11:12.582Z 15 300-319 122 122 122 208 208 208 F F F
133972024-01-24T19:11:12.582Z 16 320-339 124 124 123 212 212 209 F F F <---
133982024-01-24T19:11:12.582Z 17 340-359 122 122 122 208 208 208 F F F
133992024-01-24T19:11:12.582Z 18 360-379 123 123 123 209 209 209 T T F <---
134002024-01-24T19:11:12.582Z 19 380-399 123 123 123 209 209 209 F F F
134012024-01-24T19:11:12.582Z 20 400-419 116 116 116 200 200 200 F F F
134022024-01-24T19:11:12.582Z 21 420-439 124 124 123 211 211 209 T T F <---
134032024-01-24T19:11:12.582Z 22 440-459 120 120 120 204 204 204 F F F
134042024-01-24T19:11:12.582Z 23 460-479 123 123 123 209 209 209 F F F
134052024-01-24T19:11:12.582Z 24 480-499 122 122 122 207 207 207 F F F
134062024-01-24T19:11:12.582Z 25 500-519 124 124 123 212 212 209 F F F <---
134072024-01-24T19:11:12.582Z 26 520-539 123 123 123 209 209 209 F F F
134082024-01-24T19:11:12.582Z 27 540-559 119 119 119 203 203 203 F F F
134092024-01-24T19:11:12.583Z 28 560-579 120 120 120 205 205 205 F F F
134102024-01-24T19:11:12.583Z 29 580-599 123 123 123 209 209 209 F F F
134112024-01-24T19:11:12.583ZMax gen: 124, Max flush: 212
134122024-01-24T19:11:12.583ZError: Difference in extent metadata found!
134132024-01-24T19:11:12.587ZA Difference in extent metadata is expected here
134142024-01-24T19:11:12.587ZOn loop 62
134152024-01-24T19:11:12.587Z
134162024-01-24T19:11:12.587ZVerifying data now
134172024-01-24T19:11:18.766ZNo change after verify
134182024-01-24T19:11:18.766ZLoop: 62 Downstairs dump after verify (and repair):
134192024-01-24T19:11:18.821ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134202024-01-24T19:11:18.821Z 0 000-019 115 115 115 199 199 199 F F F
134212024-01-24T19:11:18.821Z 1 020-039 122 122 122 208 208 208 F F F
134222024-01-24T19:11:18.821Z 2 040-059 114 114 114 196 196 196 F F F
134232024-01-24T19:11:18.821Z 3 060-079 125 125 125 213 213 213 F F F
134242024-01-24T19:11:18.821Z 4 080-099 114 114 114 196 196 196 F F F
134252024-01-24T19:11:18.821Z 5 100-119 124 124 124 210 210 210 F F F
134262024-01-24T19:11:18.821Z 6 120-139 119 119 119 203 203 203 F F F
134272024-01-24T19:11:18.821Z 7 140-159 124 124 124 212 212 212 F F F
134282024-01-24T19:11:18.821Z 8 160-179 122 122 122 208 208 208 F F F
134292024-01-24T19:11:18.821Z 9 180-199 118 118 118 202 202 202 F F F
134302024-01-24T19:11:18.821Z 10 200-219 118 118 118 202 202 202 F F F
134312024-01-24T19:11:18.821Z 11 220-239 122 122 122 208 208 208 F F F
134322024-01-24T19:11:18.821Z 12 240-259 120 120 120 204 204 204 F F F
134332024-01-24T19:11:18.821Z 13 260-279 122 122 122 207 207 207 F F F
134342024-01-24T19:11:18.821Z 14 280-299 124 124 124 211 211 211 F F F
134352024-01-24T19:11:18.822Z 15 300-319 122 122 122 208 208 208 F F F
134362024-01-24T19:11:18.822Z 16 320-339 124 124 124 212 212 212 F F F
134372024-01-24T19:11:18.822Z 17 340-359 122 122 122 208 208 208 F F F
134382024-01-24T19:11:18.822Z 18 360-379 125 125 125 213 213 213 F F F
134392024-01-24T19:11:18.822Z 19 380-399 123 123 123 209 209 209 F F F
134402024-01-24T19:11:18.822Z 20 400-419 116 116 116 200 200 200 F F F
134412024-01-24T19:11:18.822Z 21 420-439 125 125 125 213 213 213 F F F
134422024-01-24T19:11:18.822Z 22 440-459 120 120 120 204 204 204 F F F
134432024-01-24T19:11:18.822Z 23 460-479 123 123 123 209 209 209 F F F
134442024-01-24T19:11:18.824Z 24 480-499 122 122 122 207 207 207 F F F
134452024-01-24T19:11:18.824Z 25 500-519 124 124 124 212 212 212 F F F
134462024-01-24T19:11:18.824Z 26 520-539 123 123 123 209 209 209 F F F
134472024-01-24T19:11:18.824Z 27 540-559 119 119 119 203 203 203 F F F
134482024-01-24T19:11:18.824Z 28 560-579 120 120 120 205 205 205 F F F
134492024-01-24T19:11:18.825Z 29 580-599 123 123 123 209 209 209 F F F
134502024-01-24T19:11:18.825ZMax gen: 125, Max flush: 213
134512024-01-24T19:11:18.825Z
134522024-01-24T19:11:18.825ZBegin loop 63
134532024-01-24T19:11:18.825ZDownstairs to restart: 0
134542024-01-24T19:11:18.825Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.092779861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633}
134552024-01-24T19:11:18.825Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093259419Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633}
134562024-01-24T19:11:18.825Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093327007Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633}
134572024-01-24T19:11:18.825Z{"msg":"Crucible e70430bc-aa33-483b-b6e0-172f3e3b6c53 has session id: ae84db04-2667-4f02-97b6-3f7cc1421c01","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093354487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134582024-01-24T19:11:18.825Z{"msg":"Upstairs opts: Upstairs UUID: e70430bc-aa33-483b-b6e0-172f3e3b6c53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093390881Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134592024-01-24T19:11:18.826Z{"msg":"Crucible stats registered with UUID: e70430bc-aa33-483b-b6e0-172f3e3b6c53","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093414289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134602024-01-24T19:11:18.826ZCrucible runtime is spawned
134612024-01-24T19:11:18.826Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093640958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633}
134622024-01-24T19:11:18.826Z{"msg":"e70430bc-aa33-483b-b6e0-172f3e3b6c53 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093884921Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134632024-01-24T19:11:18.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.093960929Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134642024-01-24T19:11:18.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094058977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134652024-01-24T19:11:18.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094101627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134662024-01-24T19:11:18.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094145599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134672024-01-24T19:11:18.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094189856Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134682024-01-24T19:11:18.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094229295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134692024-01-24T19:11:18.828Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094266741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134702024-01-24T19:11:18.828Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094309024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134712024-01-24T19:11:18.830Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094453741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134722024-01-24T19:11:18.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094642685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134732024-01-24T19:11:18.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094697943Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134742024-01-24T19:11:18.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.094742504Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134752024-01-24T19:11:18.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.095806636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134762024-01-24T19:11:18.831Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.095847213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134772024-01-24T19:11:18.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.095871031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134782024-01-24T19:11:18.831Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.095987459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134792024-01-24T19:11:18.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.096019408Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134802024-01-24T19:11:18.831Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.096056327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134812024-01-24T19:11:18.832Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.096887893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134822024-01-24T19:11:18.833Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.097204641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134832024-01-24T19:11:18.833Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.097269641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134842024-01-24T19:11:18.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.097922869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134852024-01-24T19:11:18.833Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.09803459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134862024-01-24T19:11:18.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.09807618Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134872024-01-24T19:11:18.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.09820613Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134882024-01-24T19:11:18.833Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098263493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134892024-01-24T19:11:18.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098290472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134902024-01-24T19:11:18.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.09842192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134912024-01-24T19:11:18.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.0985254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134922024-01-24T19:11:18.834Z{"msg":"[0]R flush_numbers[0..12]: [199, 208, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098571601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134932024-01-24T19:11:18.834Z{"msg":"[0]R generation[0..12]: [115, 122, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098625589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134942024-01-24T19:11:18.834Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098653573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134952024-01-24T19:11:18.834Z{"msg":"[1]R flush_numbers[0..12]: [199, 208, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098697575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134962024-01-24T19:11:18.834Z{"msg":"[1]R generation[0..12]: [115, 122, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098723759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134972024-01-24T19:11:18.834Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098774578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
134982024-01-24T19:11:18.834Z{"msg":"[2]R flush_numbers[0..12]: [199, 208, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
134992024-01-24T19:11:18.835Z,"time":"2024-01-24T19:11:16.09880187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135002024-01-24T19:11:18.835Z{"msg":"[2]R generation[0..12]: [115, 122, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098932767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135012024-01-24T19:11:18.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.098995095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135022024-01-24T19:11:18.837Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099021346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135032024-01-24T19:11:18.837Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099069444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135042024-01-24T19:11:18.837Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099094022Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135052024-01-24T19:11:18.837Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099136374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135062024-01-24T19:11:18.837Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.09916226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135072024-01-24T19:11:18.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099212275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135082024-01-24T19:11:18.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099238938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135092024-01-24T19:11:18.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099282614Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135102024-01-24T19:11:18.838Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099307843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135112024-01-24T19:11:18.839Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099354003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135122024-01-24T19:11:18.839Z{"msg":"e70430bc-aa33-483b-b6e0-172f3e3b6c53 is now active with session: ae84db04-2667-4f02-97b6-3f7cc1421c01","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099379234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135132024-01-24T19:11:18.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099421202Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633,"session_id":"ae84db04-2667-4f02-97b6-3f7cc1421c01"}
135142024-01-24T19:11:18.839Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:16.099445019Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1633}
135152024-01-24T19:11:18.839ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135162024-01-24T19:11:18.839ZLoading write count information from file "/tmp/test_repair_verify.data"
135172024-01-24T19:11:18.839ZRun Repair workload
135182024-01-24T19:11:18.839Z01/30 Write block 429 len 12288 data: 13 14 14
135192024-01-24T19:11:18.880Z02/30 Read block 491 len 28672
135202024-01-24T19:11:18.880Z03/30 Read block 459 len 28672
135212024-01-24T19:11:18.880Z04/30 Read block 189 len 4096
135222024-01-24T19:11:18.880Z05/30 Write block 552 len 40960 data: 8 7 8 6 8 10 11 10 8 9
135232024-01-24T19:11:18.880Z06/30 Read block 533 len 12288
135242024-01-24T19:11:18.880Z07/30 Flush
135252024-01-24T19:11:18.880Z08/30 Write block 572 len 36864 data: 10 8 8 9 9 4 6 8 8
135262024-01-24T19:11:18.880Z09/30 Write block 504 len 4096 data: 11
135272024-01-24T19:11:18.880Z10/30 Read block 401 len 8192
135282024-01-24T19:11:18.881Z11/30 Write block 8 len 4096 data: 12
135292024-01-24T19:11:18.881Z12/30 Read block 424 len 36864
135302024-01-24T19:11:18.881Z13/30 Write block 372 len 24576 data: 8 9 9 12 11 11
135312024-01-24T19:11:18.881Z14/30 Flush
135322024-01-24T19:11:18.881Z15/30 Write block 22 len 28672 data: 9 8 10 10 11 13 15
135332024-01-24T19:11:18.881Z16/30 Read block 24 len 28672
135342024-01-24T19:11:18.881Z17/30 Write block 559 len 4096 data: 11
135352024-01-24T19:11:18.881Z18/30 Read block 83 len 40960
135362024-01-24T19:11:18.881Z19/30 Read block 393 len 36864
135372024-01-24T19:11:18.881Z20/30 Read block 508 len 4096
135382024-01-24T19:11:18.881Z21/30 Flush
135392024-01-24T19:11:18.881Z22/30 Write block 502 len 4096 data: 11
135402024-01-24T19:11:18.882Z23/30 Flush
135412024-01-24T19:11:18.882Z24/30 Write block 240 len 32768 data: 8 9 11 14 14 15 17 16
135422024-01-24T19:11:18.882Z25/30 Read block 116 len 24576
135432024-01-24T19:11:18.882Z26/30 Read block 316 len 8192
135442024-01-24T19:11:18.882Z27/30 Write block 549 len 20480 data: 9 9 10 9 8
135452024-01-24T19:11:18.882Z28/30 Write block 530 len 12288 data: 12 10 7
135462024-01-24T19:11:18.882Z29/30 Read block 346 len 8192
135472024-01-24T19:11:18.882Z30/30 Write block 520 len 24576 data: 14 12 11 9 11 12
135482024-01-24T19:11:18.883Z----------------------------------------------------------------
135492024-01-24T19:11:18.883Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
135502024-01-24T19:11:18.883ZStates: Active Active Active
135512024-01-24T19:11:18.883ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135522024-01-24T19:11:18.883Z 1 Acked 1000 Write 3 Sent Done Done false
135532024-01-24T19:11:18.883Z 2 Acked 1001 Read 7 Sent Done Done false
135542024-01-24T19:11:18.883Z 3 Acked 1002 Read 7 Sent Done Done false
135552024-01-24T19:11:18.883Z 4 Acked 1003 Read 1 Sent Done Done false
135562024-01-24T19:11:18.883Z 5 Acked 1004 Write 10 Sent Done Done false
135572024-01-24T19:11:18.883Z 6 Acked 1005 Read 3 Sent Done Done false
135582024-01-24T19:11:18.883Z 7 Acked 1006 Flush 0 Sent Done Done false
135592024-01-24T19:11:18.883Z 8 Acked 1007 Write 9 Sent Done Done false
135602024-01-24T19:11:18.883Z 9 Acked 1008 Write 1 Sent Done Done false
135612024-01-24T19:11:18.883Z 10 Acked 1009 Read 2 Sent Done Done false
135622024-01-24T19:11:18.883Z 11 Acked 1010 Write 1 Sent Done Done false
135632024-01-24T19:11:18.883Z 12 Acked 1011 Read 9 Sent Done Done false
135642024-01-24T19:11:18.883Z 13 Acked 1012 Write 6 Sent Done Done false
135652024-01-24T19:11:18.883Z 14 Acked 1013 Flush 0 Sent Done Done false
135662024-01-24T19:11:18.883Z 15 Acked 1014 Write 7 Sent Done Done false
135672024-01-24T19:11:18.883Z 16 Acked 1015 Read 7 Sent Done Done false
135682024-01-24T19:11:18.883Z 17 Acked 1016 Write 1 Sent Done Done false
135692024-01-24T19:11:18.883Z 18 Acked 1017 Read 10 Sent Done Done false
135702024-01-24T19:11:18.884Z 19 Acked 1018 Read 9 Sent Done Done false
135712024-01-24T19:11:18.884Z 20 Acked 1019 Read 1 Sent Done Done false
135722024-01-24T19:11:18.884Z 21 Acked 1020 Flush 0 Sent Done Done false
135732024-01-24T19:11:18.884Z 22 Acked 1021 Write 1 Sent Done Done false
135742024-01-24T19:11:18.884Z 23 Acked 1022 Flush 0 Sent Done Done false
135752024-01-24T19:11:18.884Z 24 Acked 1023 Write 8 Sent Done Done false
135762024-01-24T19:11:18.884Z 25 Acked 1024 Read 6 Sent Done Done false
135772024-01-24T19:11:18.884Z 26 Acked 1025 Read 2 Sent Done Done false
135782024-01-24T19:11:18.884Z 27 Acked 1026 Write 5 Sent Done Done false
135792024-01-24T19:11:18.884Z 28 Acked 1027 Write 3 Sent Done Done false
135802024-01-24T19:11:18.884Z 29 Acked 1028 Read 2 Sent Done Done false
135812024-01-24T19:11:18.884Z 30 Acked 1029 Write 6 Sent Sent Sent false
135822024-01-24T19:11:18.884Z STATES DS:0 DS:1 DS:2 TOTAL
135832024-01-24T19:11:18.884Z New 0 0 0 0
135842024-01-24T19:11:18.884Z Sent 30 1 1 32
135852024-01-24T19:11:18.884Z Done 0 29 29 58
135862024-01-24T19:11:18.884Z Skipped 0 0 0 0
135872024-01-24T19:11:18.884Z Error 0 0 0 0
135882024-01-24T19:11:18.886ZLast Flush: 0 1022 1022
135892024-01-24T19:11:18.886ZDownstairs last five completed:
135902024-01-24T19:11:18.886ZUpstairs last five completed: 30 29 28 27 26
135912024-01-24T19:11:18.886ZWrote out file "/tmp/test_repair_verify.data"
135922024-01-24T19:11:18.886Z
135932024-01-24T19:11:18.886ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
135942024-01-24T19:11:18.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135952024-01-24T19:11:18.886Z 0 000-019 115 126 126 199 215 215 F F F <---
135962024-01-24T19:11:18.886Z 1 020-039 122 126 126 208 216 216 F F F <---
135972024-01-24T19:11:18.886Z 2 040-059 114 114 114 196 196 196 F F F
135982024-01-24T19:11:18.886Z 3 060-079 125 125 125 213 213 213 F F F
135992024-01-24T19:11:18.886Z 4 080-099 114 114 114 196 196 196 F F F
136002024-01-24T19:11:18.886Z 5 100-119 124 124 124 210 210 210 F F F
136012024-01-24T19:11:18.886Z 6 120-139 119 119 119 203 203 203 F F F
136022024-01-24T19:11:18.886Z 7 140-159 124 124 124 212 212 212 F F F
136032024-01-24T19:11:18.886Z 8 160-179 122 122 122 208 208 208 F F F
136042024-01-24T19:11:18.886Z 9 180-199 118 118 118 202 202 202 F F F
136052024-01-24T19:11:18.886Z 10 200-219 118 118 118 202 202 202 F F F
136062024-01-24T19:11:18.886Z 11 220-239 122 122 122 208 208 208 F F F
136072024-01-24T19:11:18.886Z 12 240-259 120 120 120 204 204 204 F T T <---
136082024-01-24T19:11:18.886Z 13 260-279 122 122 122 207 207 207 F F F
136092024-01-24T19:11:18.886Z 14 280-299 124 124 124 211 211 211 F F F
136102024-01-24T19:11:18.886Z 15 300-319 122 122 122 208 208 208 F F F
136112024-01-24T19:11:18.887Z 16 320-339 124 124 124 212 212 212 F F F
136122024-01-24T19:11:18.887Z 17 340-359 122 122 122 208 208 208 F F F
136132024-01-24T19:11:18.887Z 18 360-379 125 126 126 213 215 215 F F F <---
136142024-01-24T19:11:18.887Z 19 380-399 123 123 123 209 209 209 F F F
136152024-01-24T19:11:18.887Z 20 400-419 116 116 116 200 200 200 F F F
136162024-01-24T19:11:18.887Z 21 420-439 125 126 126 213 214 214 F F F <---
136172024-01-24T19:11:18.887Z 22 440-459 120 120 120 204 204 204 F F F
136182024-01-24T19:11:18.887Z 23 460-479 123 123 123 209 209 209 F F F
136192024-01-24T19:11:18.920Z 24 480-499 122 122 122 207 207 207 F F F
136202024-01-24T19:11:18.920Z 25 500-519 124 126 126 212 217 217 F F F <---
136212024-01-24T19:11:18.920Z 26 520-539 123 123 123 209 209 209 F T T <---
136222024-01-24T19:11:18.921Z 27 540-559 119 126 126 203 216 216 F T T <---
136232024-01-24T19:11:18.921Z 28 560-579 120 126 126 205 215 215 F F F <---
136242024-01-24T19:11:18.921Z 29 580-599 123 126 126 209 215 215 F F F <---
136252024-01-24T19:11:18.921ZMax gen: 126, Max flush: 217
136262024-01-24T19:11:18.921ZError: Difference in extent metadata found!
136272024-01-24T19:11:18.922ZA Difference in extent metadata is expected here
136282024-01-24T19:11:18.922ZOn loop 63
136292024-01-24T19:11:18.922Z
136302024-01-24T19:11:18.922ZVerifying data now
136312024-01-24T19:11:22.471ZNo change after verify
136322024-01-24T19:11:22.471ZLoop: 63 Downstairs dump after verify (and repair):
136332024-01-24T19:11:22.498ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136342024-01-24T19:11:22.498Z 0 000-019 126 126 126 215 215 215 F F F
136352024-01-24T19:11:22.498Z 1 020-039 126 126 126 216 216 216 F F F
136362024-01-24T19:11:22.498Z 2 040-059 114 114 114 196 196 196 F F F
136372024-01-24T19:11:22.499Z 3 060-079 125 125 125 213 213 213 F F F
136382024-01-24T19:11:22.499Z 4 080-099 114 114 114 196 196 196 F F F
136392024-01-24T19:11:22.499Z 5 100-119 124 124 124 210 210 210 F F F
136402024-01-24T19:11:22.499Z 6 120-139 119 119 119 203 203 203 F F F
136412024-01-24T19:11:22.919Z 7 140-159 124 124 124 212 212 212 F F F
136422024-01-24T19:11:22.920Z 8 160-179 122 122 122 208 208 208 F F F
136432024-01-24T19:11:22.920Z 9 180-199 118 118 118 202 202 202 F F F
136442024-01-24T19:11:22.920Z 10 200-219 118 118 118 202 202 202 F F F
136452024-01-24T19:11:22.920Z 11 220-239 122 122 122 208 208 208 F F F
136462024-01-24T19:11:22.920Z 12 240-259 127 127 127 218 218 218 F F F
136472024-01-24T19:11:22.921Z 13 260-279 122 122 122 207 207 207 F F F
136482024-01-24T19:11:22.921Z 14 280-299 124 124 124 211 211 211 F F F
136492024-01-24T19:11:22.921Z 15 300-319 122 122 122 208 208 208 F F F
136502024-01-24T19:11:22.921Z 16 320-339 124 124 124 212 212 212 F F F
136512024-01-24T19:11:22.921Z 17 340-359 122 122 122 208 208 208 F F F
136522024-01-24T19:11:22.921Z 18 360-379 126 126 126 215 215 215 F F F
136532024-01-24T19:11:22.922Z 19 380-399 123 123 123 209 209 209 F F F
136542024-01-24T19:11:22.922Z 20 400-419 116 116 116 200 200 200 F F F
136552024-01-24T19:11:22.923Z 21 420-439 126 126 126 214 214 214 F F F
136562024-01-24T19:11:22.923Z 22 440-459 120 120 120 204 204 204 F F F
136572024-01-24T19:11:22.924Z 23 460-479 123 123 123 209 209 209 F F F
136582024-01-24T19:11:22.924Z 24 480-499 122 122 122 207 207 207 F F F
136592024-01-24T19:11:22.924Z 25 500-519 126 126 126 217 217 217 F F F
136602024-01-24T19:11:22.924Z 26 520-539 127 127 127 218 218 218 F F F
136612024-01-24T19:11:22.924Z 27 540-559 127 127 127 218 218 218 F F F
136622024-01-24T19:11:22.925Z 28 560-579 126 126 126 215 215 215 F F F
136632024-01-24T19:11:22.925Z 29 580-599 126 126 126 215 215 215 F F F
136642024-01-24T19:11:22.925ZMax gen: 127, Max flush: 218
136652024-01-24T19:11:22.926Z
136662024-01-24T19:11:22.926ZBegin loop 64
136672024-01-24T19:11:22.926ZDownstairs to restart: 2
136682024-01-24T19:11:22.952Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201077083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641}
136692024-01-24T19:11:22.952Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201583133Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641}
136702024-01-24T19:11:22.952Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201614327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641}
136712024-01-24T19:11:22.952Z{"msg":"Crucible fe9db6a6-16e1-4891-8fe2-e64132611e5c has session id: 1c063e7f-f460-44c5-ac29-7e542f089378","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201637289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136722024-01-24T19:11:22.953Z{"msg":"Upstairs opts: Upstairs UUID: fe9db6a6-16e1-4891-8fe2-e64132611e5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201707331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136732024-01-24T19:11:22.953Z{"msg":"Crucible stats registered with UUID: fe9db6a6-16e1-4891-8fe2-e64132611e5c","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.201734152Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136742024-01-24T19:11:22.953ZCrucible runtime is spawned
136752024-01-24T19:11:22.953Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202013097Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641}
136762024-01-24T19:11:22.953Z{"msg":"fe9db6a6-16e1-4891-8fe2-e64132611e5c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202235557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136772024-01-24T19:11:22.953Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202285715Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136782024-01-24T19:11:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202310952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136792024-01-24T19:11:22.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202384245Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136802024-01-24T19:11:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202422246Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136812024-01-24T19:11:22.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202467794Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136822024-01-24T19:11:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202519228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136832024-01-24T19:11:22.955Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202544165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136842024-01-24T19:11:22.956Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202602004Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136852024-01-24T19:11:22.956Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202720635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136862024-01-24T19:11:22.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20287151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136872024-01-24T19:11:22.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.202963133Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136882024-01-24T19:11:22.959Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.203057226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136892024-01-24T19:11:22.959Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.204336798Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136902024-01-24T19:11:22.959Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.204446354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136912024-01-24T19:11:22.959Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.204493148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136922024-01-24T19:11:22.959Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.204532884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136932024-01-24T19:11:22.959Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.205364476Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136942024-01-24T19:11:22.959Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.205425877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136952024-01-24T19:11:22.959Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.205541923Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136962024-01-24T19:11:22.959Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.206122598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136972024-01-24T19:11:22.959Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.206533679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136982024-01-24T19:11:22.960Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20659552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
136992024-01-24T19:11:22.960Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20662275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137002024-01-24T19:11:22.960Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.206645695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137012024-01-24T19:11:22.960Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.206913275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137022024-01-24T19:11:22.960Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207000174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137032024-01-24T19:11:22.960Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20704626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137042024-01-24T19:11:22.961Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20746705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137052024-01-24T19:11:22.961Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207546987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137062024-01-24T19:11:22.961Z{"msg":"[0]R flush_numbers[0..12]: [215, 216, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.20757652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137072024-01-24T19:11:22.961Z{"msg":"[0]R generation[0..12]: [126, 126, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207601192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137082024-01-24T19:11:22.962Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207688803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137092024-01-24T19:11:22.962Z{"msg":"[1]R flush_numbers[0..12]: [215, 216, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207743295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137102024-01-24T19:11:22.962Z{"msg":"[1]R generation[0..12]: [126, 126, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207771243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137112024-01-24T19:11:22.962Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207816473Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137122024-01-24T19:11:22.962Z{"msg":"[2]R flush_numbers[0..12]: [215, 216, 196, 213, 196, 210, 203, 212, 208, 202, 202, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207843941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137132024-01-24T19:11:22.962Z{"msg":"[2]R generation[0..12]: [126, 126, 114, 125, 114, 124, 119, 124, 122, 118, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207893347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137142024-01-24T19:11:22.962Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207955616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137152024-01-24T19:11:22.962Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.207982276Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO
137162024-01-24T19:11:22.962Z"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137172024-01-24T19:11:22.962Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208112006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137182024-01-24T19:11:22.962Z{"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208141879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137192024-01-24T19:11:22.963Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208164342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137202024-01-24T19:11:22.963Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208191238Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137212024-01-24T19:11:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208216234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137222024-01-24T19:11:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208272235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137232024-01-24T19:11:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208299306Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137242024-01-24T19:11:22.963Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208321747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137252024-01-24T19:11:22.963Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208370858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137262024-01-24T19:11:22.963Z{"msg":"fe9db6a6-16e1-4891-8fe2-e64132611e5c is now active with session: 1c063e7f-f460-44c5-ac29-7e542f089378","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208396158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137272024-01-24T19:11:22.963Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208416379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641,"session_id":"1c063e7f-f460-44c5-ac29-7e542f089378"}
137282024-01-24T19:11:22.964Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:21.208458403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1641}
137292024-01-24T19:11:22.964ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137302024-01-24T19:11:22.964ZLoading write count information from file "/tmp/test_repair_verify.data"
137312024-01-24T19:11:22.965ZRun Repair workload
137322024-01-24T19:11:22.965Z01/30 Flush
137332024-01-24T19:11:22.965Z02/30 Write block 106 len 24576 data: 11 12 13 11 10 9
137342024-01-24T19:11:22.965Z03/30 Flush
137352024-01-24T19:11:22.965Z04/30 Write block 34 len 40960 data: 12 7 6 5 5 4 5 7 6 7
137362024-01-24T19:11:22.965Z05/30 Write block 47 len 28672 data: 10 11 12 14 11 10 10
137372024-01-24T19:11:22.965Z06/30 Read block 180 len 4096
137382024-01-24T19:11:22.965Z07/30 Flush
137392024-01-24T19:11:22.965Z08/30 Write block 471 len 40960 data: 11 15 12 13 14 13 12 10 7 9
137402024-01-24T19:11:22.965Z09/30 Read block 300 len 36864
137412024-01-24T19:11:22.965Z10/30 Read block 536 len 24576
137422024-01-24T19:11:22.965Z11/30 Write block 537 len 24576 data: 10 10 10 8 9 12
137432024-01-24T19:11:22.965Z12/30 Read block 473 len 12288
137442024-01-24T19:11:22.965Z13/30 Read block 7 len 12288
137452024-01-24T19:11:22.965Z14/30 Read block 566 len 36864
137462024-01-24T19:11:22.965Z15/30 Write block 287 len 12288 data: 9 11 10
137472024-01-24T19:11:22.965Z16/30 Read block 585 len 28672
137482024-01-24T19:11:22.965Z17/30 Read block 215 len 8192
137492024-01-24T19:11:22.965Z18/30 Write block 280 len 12288 data: 13 13 14
137502024-01-24T19:11:22.965Z19/30 Read block 367 len 12288
137512024-01-24T19:11:22.966Z20/30 Write block 355 len 16384 data: 10 12 10 12
137522024-01-24T19:11:22.967Z21/30 Write block 194 len 32768 data: 10 9 10 10 9 7 7 7
137532024-01-24T19:11:22.967Z22/30 Read block 264 len 24576
137542024-01-24T19:11:22.967Z23/30 Read block 151 len 20480
137552024-01-24T19:11:22.968Z24/30 Read block 223 len 20480
137562024-01-24T19:11:22.968Z25/30 Write block 143 len 24576 data: 6 8 9 11 13 12
137572024-01-24T19:11:22.968Z26/30 Read block 166 len 24576
137582024-01-24T19:11:22.968Z27/30 Read block 327 len 36864
137592024-01-24T19:11:22.968Z28/30 Read block 451 len 20480
137602024-01-24T19:11:22.968Z29/30 Read block 110 len 20480
137612024-01-24T19:11:22.968Z30/30 Write block 584 len 20480 data: 10 8 9 9 9
137622024-01-24T19:11:22.968Z----------------------------------------------------------------
137632024-01-24T19:11:22.968Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:27
137642024-01-24T19:11:22.968ZStates: Active Active Active
137652024-01-24T19:11:22.968ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137662024-01-24T19:11:22.968Z 4 Acked 1003 Write 10 Done Done Sent false
137672024-01-24T19:11:22.968Z 5 Acked 1004 Write 7 Done Done Sent false
137682024-01-24T19:11:22.977Z 6 Acked 1005 Read 1 Done Done Sent false
137692024-01-24T19:11:22.977Z 7 Acked 1006 Flush 0 Done Done Sent false
137702024-01-24T19:11:22.977Z 8 Acked 1007 Write 10 Done Done Sent false
137712024-01-24T19:11:22.977Z 9 Acked 1008 Read 9 Done Done Sent false
137722024-01-24T19:11:22.977Z 10 Acked 1009 Read 6 Done Done Sent false
137732024-01-24T19:11:22.977Z 11 Acked 1010 Write 6 Done Done Sent false
137742024-01-24T19:11:22.977Z 12 Acked 1011 Read 3 Done Done Sent false
137752024-01-24T19:11:22.979Z 13 Acked 1012 Read 3 Done Done Sent false
137762024-01-24T19:11:22.979Z 14 Acked 1013 Read 9 Done Done Sent false
137772024-01-24T19:11:22.979Z 15 Acked 1014 Write 3 Done Done Sent false
137782024-01-24T19:11:22.979Z 16 Acked 1015 Read 7 Done Done Sent false
137792024-01-24T19:11:22.979Z 17 Acked 1016 Read 2 Done Done Sent false
137802024-01-24T19:11:22.979Z 18 Acked 1017 Write 3 Done Done Sent false
137812024-01-24T19:11:22.979Z 19 Acked 1018 Read 3 Done Done Sent false
137822024-01-24T19:11:22.979Z 20 Acked 1019 Write 4 Done Done Sent false
137832024-01-24T19:11:22.979Z 21 Acked 1020 Write 8 Done Done Sent false
137842024-01-24T19:11:22.979Z 22 Acked 1021 Read 6 Done Done Sent false
137852024-01-24T19:11:22.979Z 23 Acked 1022 Read 5 Done Done Sent false
137862024-01-24T19:11:22.980Z 24 Acked 1023 Read 5 Done Done Sent false
137872024-01-24T19:11:22.980Z 25 Acked 1024 Write 6 Done Done Sent false
137882024-01-24T19:11:22.980Z 26 Acked 1025 Read 6 Done Done Sent false
137892024-01-24T19:11:22.980Z 27 Acked 1026 Read 9 Done Done Sent false
137902024-01-24T19:11:22.980Z 28 Acked 1027 Read 5 Done Done Sent false
137912024-01-24T19:11:22.980Z 29 Acked 1028 Read 5 Done Done Sent false
137922024-01-24T19:11:22.980Z 30 Acked 1029 Write 5 Sent Sent Sent false
137932024-01-24T19:11:22.980Z STATES DS:0 DS:1 DS:2 TOTAL
137942024-01-24T19:11:22.980Z New 0 0 0 0
137952024-01-24T19:11:22.980Z Sent 1 1 27 29
137962024-01-24T19:11:22.980Z Done 26 26 0 52
137972024-01-24T19:11:22.980Z Skipped 0 0 0 0
137982024-01-24T19:11:22.981Z Error 0 0 0 0
137992024-01-24T19:11:22.981ZLast Flush: 1006 1006 1002
138002024-01-24T19:11:22.981ZDownstairs last five completed: 1002 1001 1000
138012024-01-24T19:11:22.981ZUpstairs last five completed: 30 29 28 27 26
138022024-01-24T19:11:22.981ZWrote out file "/tmp/test_repair_verify.data"
138032024-01-24T19:11:22.981Z
138042024-01-24T19:11:22.981ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138052024-01-24T19:11:22.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138062024-01-24T19:11:22.981Z 0 000-019 126 126 126 215 215 215 F F F
138072024-01-24T19:11:22.982Z 1 020-039 128 128 126 221 221 216 F F F <---
138082024-01-24T19:11:22.982Z 2 040-059 128 128 114 221 221 196 F F F <---
138092024-01-24T19:11:22.982Z 3 060-079 125 125 125 213 213 213 F F F
138102024-01-24T19:11:22.982Z 4 080-099 114 114 114 196 196 196 F F F
138112024-01-24T19:11:22.982Z 5 100-119 128 128 128 220 220 220 F F F
138122024-01-24T19:11:22.982Z 6 120-139 119 119 119 203 203 203 F F F
138132024-01-24T19:11:22.982Z 7 140-159 124 124 124 212 212 212 T T F <---
138142024-01-24T19:11:22.982Z 8 160-179 122 122 122 208 208 208 F F F
138152024-01-24T19:11:22.982Z 9 180-199 118 118 118 202 202 202 T T F <---
138162024-01-24T19:11:22.982Z 10 200-219 118 118 118 202 202 202 T T F <---
138172024-01-24T19:11:22.982Z 11 220-239 122 122 122 208 208 208 F F F
138182024-01-24T19:11:22.982Z 12 240-259 127 127 127 218 218 218 F F F
138192024-01-24T19:11:22.982Z 13 260-279 122 122 122 207 207 207 F F F
138202024-01-24T19:11:22.983Z 14 280-299 124 124 124 211 211 211 T T F <---
138212024-01-24T19:11:22.983Z 15 300-319 122 122 122 208 208 208 F F F
138222024-01-24T19:11:22.983Z 16 320-339 124 124 124 212 212 212 F F F
138232024-01-24T19:11:22.983Z 17 340-359 122 122 122 208 208 208 T T F <---
138242024-01-24T19:11:22.983Z 18 360-379 126 126 126 215 215 215 F F F
138252024-01-24T19:11:22.983Z 19 380-399 123 123 123 209 209 209 F F F
138262024-01-24T19:11:22.983Z 20 400-419 116 116 116 200 200 200 F F F
138272024-01-24T19:11:22.983Z 21 420-439 126 126 126 214 214 214 F F F
138282024-01-24T19:11:22.983Z 22 440-459 120 120 120 204 204 204 F F F
138292024-01-24T19:11:22.983Z 23 460-479 123 123 123 209 209 209 T T F <---
138302024-01-24T19:11:22.984Z 24 480-499 122 122 122 207 207 207 T T F <---
138312024-01-24T19:11:22.984Z 25 500-519 126 126 126 217 217 217 F F F
138322024-01-24T19:11:22.984Z 26 520-539 127 127 127 218 218 218 T T F <---
138332024-01-24T19:11:22.984Z 27 540-559 127 127 127 218 218 218 T T F <---
138342024-01-24T19:11:22.984Z 28 560-579 126 126 126 215 215 215 F F F
138352024-01-24T19:11:22.984Z 29 580-599 126 126 126 215 215 215 T T F <---
138362024-01-24T19:11:22.984ZMax gen: 128, Max flush: 221
138372024-01-24T19:11:22.984ZError: Difference in extent metadata found!
138382024-01-24T19:11:22.984ZA Difference in extent metadata is expected here
138392024-01-24T19:11:22.984ZOn loop 64
138402024-01-24T19:11:22.984Z
138412024-01-24T19:11:22.984ZVerifying data now
138422024-01-24T19:11:27.742ZNo change after verify
138432024-01-24T19:11:27.742ZLoop: 64 Downstairs dump after verify (and repair):
138442024-01-24T19:11:27.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138452024-01-24T19:11:27.767Z 0 000-019 126 126 126 215 215 215 F F F
138462024-01-24T19:11:27.771Z 1 020-039 128 128 128 221 221 221 F F F
138472024-01-24T19:11:27.772Z 2 040-059 128 128 128 221 221 221 F F F
138482024-01-24T19:11:27.772Z 3 060-079 125 125 125 213 213 213 F F F
138492024-01-24T19:11:27.772Z 4 080-099 114 114 114 196 196 196 F F F
138502024-01-24T19:11:27.772Z 5 100-119 128 128 128 220 220 220 F F F
138512024-01-24T19:11:27.772Z 6 120-139 119 119 119 203 203 203 F F F
138522024-01-24T19:11:27.772Z 7 140-159 129 129 129 222 222 222 F F F
138532024-01-24T19:11:27.772Z 8 160-179 122 122 122 208 208 208 F F F
138542024-01-24T19:11:27.772Z 9 180-199 129 129 129 222 222 222 F F F
138552024-01-24T19:11:27.772Z 10 200-219 129 129 129 222 222 222 F F F
138562024-01-24T19:11:27.772Z 11 220-239 122 122 122 208 208 208 F F F
138572024-01-24T19:11:27.773Z 12 240-259 127 127 127 218 218 218 F F F
138582024-01-24T19:11:27.773Z 13 260-279 122 122 122 207 207 207 F F F
138592024-01-24T19:11:27.773Z 14 280-299 129 129 129 222 222 222 F F F
138602024-01-24T19:11:27.773Z 15 300-319 122 122 122 208 208 208 F F F
138612024-01-24T19:11:27.773Z 16 320-339 124 124 124 212 212 212 F F F
138622024-01-24T19:11:27.773Z 17 340-359 129 129 129 222 222 222 F F F
138632024-01-24T19:11:27.773Z 18 360-379 126 126 126 215 215 215 F F F
138642024-01-24T19:11:27.773Z 19 380-399 123 123 123 209 209 209 F F F
138652024-01-24T19:11:27.773Z 20 400-419 116 116 116 200 200 200 F F F
138662024-01-24T19:11:27.773Z 21 420-439 126 126 126 214 214 214 F F F
138672024-01-24T19:11:27.773Z 22 440-459 120 120 120 204 204 204 F F F
138682024-01-24T19:11:27.773Z 23 460-479 129 129 129 222 222 222 F F F
138692024-01-24T19:11:27.773Z 24 480-499 129 129 129 222 222 222 F F F
138702024-01-24T19:11:27.773Z 25 500-519 126 126 126 217 217 217 F F F
138712024-01-24T19:11:27.773Z 26 520-539 129 129 129 222 222 222 F F F
138722024-01-24T19:11:27.773Z 27 540-559 129 129 129 222 222 222 F F F
138732024-01-24T19:11:27.773Z 28 560-579 126 126 126 215 215 215 F F F
138742024-01-24T19:11:27.773Z 29 580-599 129 129 129 222 222 222 F F F
138752024-01-24T19:11:27.774ZMax gen: 129, Max flush: 222
138762024-01-24T19:11:27.774Z
138772024-01-24T19:11:27.774ZBegin loop 65
138782024-01-24T19:11:27.774ZDownstairs to restart: 0
138792024-01-24T19:11:27.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.469632129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649}
138802024-01-24T19:11:27.805Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470106164Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649}
138812024-01-24T19:11:27.805Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470135464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649}
138822024-01-24T19:11:27.805Z{"msg":"Crucible b9835119-0252-4c74-b0e3-36c1235bb5be has session id: 40cd6d18-8d2a-486f-9b42-ce22b06fb5b4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470188342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138832024-01-24T19:11:27.810Z{"msg":"Upstairs opts: Upstairs UUID: b9835119-0252-4c74-b0e3-36c1235bb5be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.47024072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138842024-01-24T19:11:27.810ZCrucible runtime is spawned
138852024-01-24T19:11:27.811Z{"msg":"Crucible stats registered with UUID: b9835119-0252-4c74-b0e3-36c1235bb5be","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470418474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138862024-01-24T19:11:27.811Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470502317Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649}
138872024-01-24T19:11:27.811Z{"msg":"b9835119-0252-4c74-b0e3-36c1235bb5be active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470837269Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138882024-01-24T19:11:27.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.470903874Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138892024-01-24T19:11:27.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471018235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138902024-01-24T19:11:27.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471107708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138912024-01-24T19:11:27.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471154555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138922024-01-24T19:11:27.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.47125147Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138932024-01-24T19:11:27.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471684995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138942024-01-24T19:11:27.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471731464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138952024-01-24T19:11:27.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471801522Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138962024-01-24T19:11:27.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471871468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138972024-01-24T19:11:27.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.471940845Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138982024-01-24T19:11:27.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.472036703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
138992024-01-24T19:11:27.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.472119497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139002024-01-24T19:11:27.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.472681395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139012024-01-24T19:11:27.812Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.472743127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139022024-01-24T19:11:27.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.472995263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139032024-01-24T19:11:27.812Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.473053212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139042024-01-24T19:11:27.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.473156506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139052024-01-24T19:11:27.812Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.473182303Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139062024-01-24T19:11:27.812Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.473879215Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139072024-01-24T19:11:27.812Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474156573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139082024-01-24T19:11:27.813Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474446259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139092024-01-24T19:11:27.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474876927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139102024-01-24T19:11:27.813Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474937205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139112024-01-24T19:11:27.813Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474964412Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139122024-01-24T19:11:27.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.474985571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139132024-01-24T19:11:27.813Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475031191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139142024-01-24T19:11:27.813Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475055846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139152024-01-24T19:11:27.813Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475271592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139162024-01-24T19:11:27.813Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475369604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139172024-01-24T19:11:27.813Z{"msg":"[0]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 220, 203, 222, 208, 222, 222, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475437543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139182024-01-24T19:11:27.814Z{"msg":"[0]R generation[0..12]: [126, 128, 128, 125, 114, 128, 119, 129, 122, 129, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475466241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139192024-01-24T19:11:27.814Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475512369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139202024-01-24T19:11:27.814Z{"msg":"[1]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 220, 203, 222, 208, 222, 222, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475538997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139212024-01-24T19:11:27.814Z{"msg":"[1]R generation[0..12]: [126, 128, 128, 125, 114, 128, 119, 129, 122, 129, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475583292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139222024-01-24T19:11:27.814Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475609747Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139232024-01-24T19:11:27.814Z{"msg":"[2]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 220, 203, 222, 208, 222, 222, 208]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475670947Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139242024-01-24T19:11:27.814Z{"msg":"[2]R generation[0..12]: [126, 128, 128, 125, 114, 128, 119, 129, 122, 129, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475697487Z","hostname":"Wait for a query_work_queue command to finish before sending IO
139252024-01-24T19:11:27.814Zip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139262024-01-24T19:11:27.815Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475724354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139272024-01-24T19:11:27.816Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475745737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139282024-01-24T19:11:27.816Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475767435Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139292024-01-24T19:11:27.816Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475831709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139302024-01-24T19:11:27.816Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475858798Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139312024-01-24T19:11:27.816Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475911925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139322024-01-24T19:11:27.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475935431Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139332024-01-24T19:11:27.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.475979777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139342024-01-24T19:11:27.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476054719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139352024-01-24T19:11:27.816Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476081182Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139362024-01-24T19:11:27.816Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476101679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139372024-01-24T19:11:27.817Z{"msg":"b9835119-0252-4c74-b0e3-36c1235bb5be is now active with session: 40cd6d18-8d2a-486f-9b42-ce22b06fb5b4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476145181Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139382024-01-24T19:11:27.817Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476188646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649,"session_id":"40cd6d18-8d2a-486f-9b42-ce22b06fb5b4"}
139392024-01-24T19:11:27.817Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:26.476212228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1649}
139402024-01-24T19:11:27.817ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
139412024-01-24T19:11:27.817ZLoading write count information from file "/tmp/test_repair_verify.data"
139422024-01-24T19:11:27.817ZRun Repair workload
139432024-01-24T19:11:27.818Z01/30 Write block 340 len 32768 data: 10 9 8 12 10 10 8 8
139442024-01-24T19:11:27.822Z02/30 Write block 574 len 28672 data: 9 10 10 5 7 9 9
139452024-01-24T19:11:27.822Z03/30 Write block 185 len 28672 data: 11 10 9 8 7 7 7
139462024-01-24T19:11:27.822Z04/30 Flush
139472024-01-24T19:11:27.823Z05/30 Write block 571 len 8192 data: 9 11
139482024-01-24T19:11:27.823Z06/30 Read block 49 len 24576
139492024-01-24T19:11:27.823Z07/30 Read block 99 len 20480
139502024-01-24T19:11:27.823Z08/30 Flush
139512024-01-24T19:11:27.823Z09/30 Write block 488 len 16384 data: 6 8 9 8
139522024-01-24T19:11:27.823Z10/30 Read block 239 len 4096
139532024-01-24T19:11:27.826Z11/30 Read block 555 len 36864
139542024-01-24T19:11:27.826Z12/30 Read block 50 len 8192
139552024-01-24T19:11:27.829Z13/30 Write block 247 len 36864 data: 17 14 16 13 11 7 6 7 9
139562024-01-24T19:11:27.829Z14/30 Read block 529 len 36864
139572024-01-24T19:11:27.832Z15/30 Read block 3 len 12288
139582024-01-24T19:11:27.832Z16/30 Flush
139592024-01-24T19:11:27.835Z17/30 Write block 305 len 24576 data: 6 6 7 5 5 6
139602024-01-24T19:11:27.838Z18/30 Write block 236 len 12288 data: 8 7 6
139612024-01-24T19:11:27.838Z19/30 Write block 152 len 28672 data: 14 13 13 11 11 13 13
139622024-01-24T19:11:27.838Z20/30 Read block 117 len 8192
139632024-01-24T19:11:27.841Z21/30 Read block 188 len 8192
139642024-01-24T19:11:27.841Z22/30 Read block 102 len 24576
139652024-01-24T19:11:27.844Z23/30 Read block 476 len 40960
139662024-01-24T19:11:27.844Z24/30 Read block 567 len 24576
139672024-01-24T19:11:27.847Z25/30 Read block 318 len 24576
139682024-01-24T19:11:27.850Z26/30 Flush
139692024-01-24T19:11:27.853Z27/30 Write block 596 len 4096 data: 9
139702024-01-24T19:11:27.853Z28/30 Write block 144 len 4096 data: 9
139712024-01-24T19:11:27.853Z29/30 Read block 80 len 24576
139722024-01-24T19:11:27.856Z30/30 Write block 111 len 24576 data: 10 9 7 4 5 5
139732024-01-24T19:11:27.857Z----------------------------------------------------------------
139742024-01-24T19:11:27.857Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
139752024-01-24T19:11:27.857ZStates: Active Active Active
139762024-01-24T19:11:27.857ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
139772024-01-24T19:11:27.857Z 1 Acked 1000 Write 8 Done Done Done false
139782024-01-24T19:11:27.857Z 2 Acked 1001 Write 7 Done Done Done false
139792024-01-24T19:11:27.857Z 3 Acked 1002 Write 7 Done Done Done false
139802024-01-24T19:11:27.857Z 4 Acked 1003 Flush 0 Sent Done Done false
139812024-01-24T19:11:27.857Z 5 Acked 1004 Write 2 Sent Done Done false
139822024-01-24T19:11:27.857Z 6 Acked 1005 Read 6 Sent Done Done false
139832024-01-24T19:11:27.860Z 7 Acked 1006 Read 5 Sent Done Done false
139842024-01-24T19:11:27.860Z 8 Acked 1007 Flush 0 Sent Done Done false
139852024-01-24T19:11:27.860Z 9 Acked 1008 Write 4 Sent Done Done false
139862024-01-24T19:11:27.860Z 10 Acked 1009 Read 1 Sent Done Done false
139872024-01-24T19:11:27.860Z 11 Acked 1010 Read 9 Sent Done Done false
139882024-01-24T19:11:27.860Z 12 Acked 1011 Read 2 Sent Done Done false
139892024-01-24T19:11:27.860Z 13 Acked 1012 Write 9 Sent Done Done false
139902024-01-24T19:11:27.860Z 14 Acked 1013 Read 9 Sent Done Done false
139912024-01-24T19:11:27.860Z 15 Acked 1014 Read 3 Sent Done Done false
139922024-01-24T19:11:27.860Z 16 Acked 1015 Flush 0 Sent Done Done false
139932024-01-24T19:11:27.860Z 17 Acked 1016 Write 6 Sent Done Done false
139942024-01-24T19:11:27.860Z 18 Acked 1017 Write 3 Sent Done Done false
139952024-01-24T19:11:27.860Z 19 Acked 1018 Write 7 Sent Done Done false
139962024-01-24T19:11:27.860Z 20 Acked 1019 Read 2 Sent Done Done false
139972024-01-24T19:11:27.860Z 21 Acked 1020 Read 2 Sent Done Done false
139982024-01-24T19:11:27.860Z 22 Acked 1021 Read 6 Sent Done Done false
139992024-01-24T19:11:27.860Z 23 Acked 1022 Read 10 Sent Done Done false
140002024-01-24T19:11:27.860Z 24 Acked 1023 Read 6 Sent Done Done false
140012024-01-24T19:11:27.860Z 25 Acked 1024 Read 6 Sent Done Done false
140022024-01-24T19:11:27.860Z 26 Acked 1025 Flush 0 Sent Done Done false
140032024-01-24T19:11:27.860Z 27 Acked 1026 Write 1 Sent Done Done false
140042024-01-24T19:11:27.860Z 28 Acked 1027 Write 1 Sent Done Done false
140052024-01-24T19:11:27.861Z 29 Acked 1028 Read 6 Sent Done Done false
140062024-01-24T19:11:27.861Z 30 Acked 1029 Write 6 Sent Sent Sent false
140072024-01-24T19:11:27.861Z STATES DS:0 DS:1 DS:2 TOTAL
140082024-01-24T19:11:27.861Z New 0 0 0 0
140092024-01-24T19:11:27.861Z Sent 27 1 1 29
140102024-01-24T19:11:27.861Z Done 3 29 29 61
140112024-01-24T19:11:27.861Z Skipped 0 0 0 0
140122024-01-24T19:11:27.861Z Error 0 0 0 0
140132024-01-24T19:11:27.861ZLast Flush: 0 1025 1025
140142024-01-24T19:11:27.861ZDownstairs last five completed:
140152024-01-24T19:11:27.862ZUpstairs last five completed: 30 29 28 27 26
140162024-01-24T19:11:27.862ZWrote out file "/tmp/test_repair_verify.data"
140172024-01-24T19:11:27.865Z
140182024-01-24T19:11:27.868ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140192024-01-24T19:11:27.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140202024-01-24T19:11:27.892Z 0 000-019 126 126 126 215 215 215 F F F
140212024-01-24T19:11:27.892Z 1 020-039 128 128 128 221 221 221 F F F
140222024-01-24T19:11:27.892Z 2 040-059 128 128 128 221 221 221 F F F
140232024-01-24T19:11:27.892Z 3 060-079 125 125 125 213 213 213 F F F
140242024-01-24T19:11:27.892Z 4 080-099 114 114 114 196 196 196 F F F
140252024-01-24T19:11:27.892Z 5 100-119 128 128 128 220 220 220 F T T <---
140262024-01-24T19:11:27.892Z 6 120-139 119 119 119 203 203 203 F F F
140272024-01-24T19:11:27.892Z 7 140-159 129 130 130 222 226 226 F T T <---
140282024-01-24T19:11:27.895Z 8 160-179 122 122 122 208 208 208 F F F
140292024-01-24T19:11:27.895Z 9 180-199 129 130 130 222 223 223 T F F <---
140302024-01-24T19:11:27.896Z 10 200-219 129 129 129 222 222 222 F F F
140312024-01-24T19:11:27.896Z 11 220-239 122 130 130 208 226 226 F F F <---
140322024-01-24T19:11:27.896Z 12 240-259 127 130 130 218 225 225 F F F <---
140332024-01-24T19:11:27.896Z 13 260-279 122 122 122 207 207 207 F F F
140342024-01-24T19:11:27.896Z 14 280-299 129 129 129 222 222 222 F F F
140352024-01-24T19:11:27.896Z 15 300-319 122 130 130 208 226 226 F F F <---
140362024-01-24T19:11:27.896Z 16 320-339 124 124 124 212 212 212 F F F
140372024-01-24T19:11:27.896Z 17 340-359 129 130 130 222 223 223 T F F <---
140382024-01-24T19:11:27.896Z 18 360-379 126 126 126 215 215 215 F F F
140392024-01-24T19:11:27.896Z 19 380-399 123 123 123 209 209 209 F F F
140402024-01-24T19:11:27.896Z 20 400-419 116 116 116 200 200 200 F F F
140412024-01-24T19:11:27.896Z 21 420-439 126 126 126 214 214 214 F F F
140422024-01-24T19:11:27.896Z 22 440-459 120 120 120 204 204 204 F F F
140432024-01-24T19:11:27.896Z 23 460-479 129 129 129 222 222 222 F F F
140442024-01-24T19:11:27.896Z 24 480-499 129 130 130 222 225 225 F F F <---
140452024-01-24T19:11:27.896Z 25 500-519 126 126 126 217 217 217 F F F
140462024-01-24T19:11:27.896Z 26 520-539 129 129 129 222 222 222 F F F
140472024-01-24T19:11:27.896Z 27 540-559 129 129 129 222 222 222 F F F
140482024-01-24T19:11:27.896Z 28 560-579 126 130 130 215 224 224 T F F <---
140492024-01-24T19:11:27.896Z 29 580-599 129 130 130 222 223 223 T T T <---
140502024-01-24T19:11:27.896ZMax gen: 130, Max flush: 226
140512024-01-24T19:11:27.897ZError: Difference in extent metadata found!
140522024-01-24T19:11:27.902ZA Difference in extent metadata is expected here
140532024-01-24T19:11:27.902ZOn loop 65
140542024-01-24T19:11:27.902Z
140552024-01-24T19:11:27.903ZVerifying data now
140562024-01-24T19:11:33.834ZNo change after verify
140572024-01-24T19:11:33.834ZLoop: 65 Downstairs dump after verify (and repair):
140582024-01-24T19:11:33.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140592024-01-24T19:11:33.856Z 0 000-019 126 126 126 215 215 215 F F F
140602024-01-24T19:11:33.856Z 1 020-039 128 128 128 221 221 221 F F F
140612024-01-24T19:11:33.856Z 2 040-059 128 128 128 221 221 221 F F F
140622024-01-24T19:11:33.856Z 3 060-079 125 125 125 213 213 213 F F F
140632024-01-24T19:11:33.856Z 4 080-099 114 114 114 196 196 196 F F F
140642024-01-24T19:11:33.856Z 5 100-119 131 131 131 227 227 227 F F F
140652024-01-24T19:11:33.860Z 6 120-139 119 119 119 203 203 203 F F F
140662024-01-24T19:11:33.860Z 7 140-159 131 131 131 227 227 227 F F F
140672024-01-24T19:11:33.860Z 8 160-179 122 122 122 208 208 208 F F F
140682024-01-24T19:11:33.860Z 9 180-199 130 130 130 223 223 223 F F F
140692024-01-24T19:11:33.860Z 10 200-219 129 129 129 222 222 222 F F F
140702024-01-24T19:11:33.860Z 11 220-239 130 130 130 226 226 226 F F F
140712024-01-24T19:11:33.860Z 12 240-259 130 130 130 225 225 225 F F F
140722024-01-24T19:11:33.860Z 13 260-279 122 122 122 207 207 207 F F F
140732024-01-24T19:11:33.860Z 14 280-299 129 129 129 222 222 222 F F F
140742024-01-24T19:11:33.862Z 15 300-319 130 130 130 226 226 226 F F F
140752024-01-24T19:11:33.862Z 16 320-339 124 124 124 212 212 212 F F F
140762024-01-24T19:11:33.862Z 17 340-359 130 130 130 223 223 223 F F F
140772024-01-24T19:11:33.862Z 18 360-379 126 126 126 215 215 215 F F F
140782024-01-24T19:11:33.862Z 19 380-399 123 123 123 209 209 209 F F F
140792024-01-24T19:11:33.862Z 20 400-419 116 116 116 200 200 200 F F F
140802024-01-24T19:11:33.862Z 21 420-439 126 126 126 214 214 214 F F F
140812024-01-24T19:11:33.862Z 22 440-459 120 120 120 204 204 204 F F F
140822024-01-24T19:11:33.862Z 23 460-479 129 129 129 222 222 222 F F F
140832024-01-24T19:11:33.862Z 24 480-499 130 130 130 225 225 225 F F F
140842024-01-24T19:11:33.862Z 25 500-519 126 126 126 217 217 217 F F F
140852024-01-24T19:11:33.862Z 26 520-539 129 129 129 222 222 222 F F F
140862024-01-24T19:11:33.862Z 27 540-559 129 129 129 222 222 222 F F F
140872024-01-24T19:11:33.862Z 28 560-579 130 130 130 224 224 224 F F F
140882024-01-24T19:11:33.862Z 29 580-599 131 131 131 227 227 227 F F F
140892024-01-24T19:11:33.862ZMax gen: 131, Max flush: 227
140902024-01-24T19:11:33.866Z
140912024-01-24T19:11:33.866ZBegin loop 66
140922024-01-24T19:11:33.866ZDownstairs to restart: 1
140932024-01-24T19:11:33.929Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.565902818Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657}
140942024-01-24T19:11:33.929Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566408789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657}
140952024-01-24T19:11:35.316Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566486235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657}
140962024-01-24T19:11:35.316Z{"msg":"Crucible df4f88d4-23a9-458f-95f7-ed4f94c64a91 has session id: 6d79d564-2722-4713-9eaf-8c0fc903d0eb","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566541632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
140972024-01-24T19:11:35.317Z{"msg":"Upstairs opts: Upstairs UUID: df4f88d4-23a9-458f-95f7-ed4f94c64a91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566593653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
140982024-01-24T19:11:35.317Z{"msg":"Crucible stats registered with UUID: df4f88d4-23a9-458f-95f7-ed4f94c64a91","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566631821Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
140992024-01-24T19:11:35.317ZCrucible runtime is spawned
141002024-01-24T19:11:35.317Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.566923817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657}
141012024-01-24T19:11:35.318Z{"msg":"df4f88d4-23a9-458f-95f7-ed4f94c64a91 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.56717679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141022024-01-24T19:11:35.318Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567239096Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141032024-01-24T19:11:35.318Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567267498Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141042024-01-24T19:11:35.318Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567346652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141052024-01-24T19:11:35.319Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567417949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141062024-01-24T19:11:35.319Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.56744676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141072024-01-24T19:11:35.320Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.56747263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141082024-01-24T19:11:35.320Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.56751364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141092024-01-24T19:11:35.320Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567616253Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141102024-01-24T19:11:35.320Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567666066Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141112024-01-24T19:11:35.320Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567711275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141122024-01-24T19:11:35.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567757423Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141132024-01-24T19:11:35.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.567822932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141142024-01-24T19:11:35.322Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.569185707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141152024-01-24T19:11:35.323Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.569331286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141162024-01-24T19:11:35.323Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.569472679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141172024-01-24T19:11:35.323Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.569580974Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141182024-01-24T19:11:35.323Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.5696286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141192024-01-24T19:11:35.324Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.569670198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141202024-01-24T19:11:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.570207752Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141212024-01-24T19:11:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.5704295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141222024-01-24T19:11:35.324Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.570575752Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141232024-01-24T19:11:35.324Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.570991998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141242024-01-24T19:11:35.324Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.57106116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141252024-01-24T19:11:35.326Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571090422Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141262024-01-24T19:11:35.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571513122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141272024-01-24T19:11:35.326Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571576207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141282024-01-24T19:11:35.326Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.57160392Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141292024-01-24T19:11:35.326Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571655051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141302024-01-24T19:11:35.326Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571685217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141312024-01-24T19:11:35.327Z{"msg":"[0]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 227, 203, 227, 208, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571734054Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141322024-01-24T19:11:35.327Z{"msg":"[0]R generation[0..12]: [126, 128, 128, 125, 114, 131, 119, 131, 122, 130, 129, 130]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571766992Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141332024-01-24T19:11:35.327Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571814916Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141342024-01-24T19:11:35.327Z{"msg":"[1]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 227, 203, 227, 208, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571848086Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141352024-01-24T19:11:35.329Z{"msg":"[1]R generation[0..12]: [126, 128, 128, 125, 114, 131, 119, 131, 122, 130, 129, 130]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571895343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
141362024-01-24T19:11:35.329Z,"pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141372024-01-24T19:11:35.330Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.571969117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141382024-01-24T19:11:35.330Z{"msg":"[2]R flush_numbers[0..12]: [215, 221, 221, 213, 196, 227, 203, 227, 208, 223, 222, 226]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572002341Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141392024-01-24T19:11:35.330Z{"msg":"[2]R generation[0..12]: [126, 128, 128, 125, 114, 131, 119, 131, 122, 130, 129, 130]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572057169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141402024-01-24T19:11:35.330Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572085201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141412024-01-24T19:11:35.330Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572132702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141422024-01-24T19:11:35.330Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572158786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141432024-01-24T19:11:35.331Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572214347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141442024-01-24T19:11:35.331Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572277074Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141452024-01-24T19:11:35.331Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572304298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141462024-01-24T19:11:35.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572326167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141472024-01-24T19:11:35.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572369498Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141482024-01-24T19:11:35.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572394474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141492024-01-24T19:11:35.331Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572437065Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141502024-01-24T19:11:35.332Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572460696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141512024-01-24T19:11:35.332Z{"msg":"df4f88d4-23a9-458f-95f7-ed4f94c64a91 is now active with session: 6d79d564-2722-4713-9eaf-8c0fc903d0eb","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.57250814Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141522024-01-24T19:11:35.332ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141532024-01-24T19:11:35.332Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.572535564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657,"session_id":"6d79d564-2722-4713-9eaf-8c0fc903d0eb"}
141542024-01-24T19:11:35.332Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:32.57257689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1657}
141552024-01-24T19:11:35.332ZLoading write count information from file "/tmp/test_repair_verify.data"
141562024-01-24T19:11:35.333ZRun Repair workload
141572024-01-24T19:11:35.333Z01/30 Write block 103 len 32768 data: 10 11 12 12 13 14 12 11
141582024-01-24T19:11:35.333Z02/30 Flush
141592024-01-24T19:11:35.333Z03/30 Write block 186 len 40960 data: 11 10 9 8 8 8 9 10 11 10
141602024-01-24T19:11:35.333Z04/30 Read block 72 len 28672
141612024-01-24T19:11:35.333Z05/30 Write block 97 len 24576 data: 10 9 7 7 5 6
141622024-01-24T19:11:35.333Z06/30 Read block 533 len 36864
141632024-01-24T19:11:35.334Z07/30 Write block 561 len 36864 data: 10 7 7 6 9 9 12 14 13
141642024-01-24T19:11:35.334Z08/30 Read block 86 len 20480
141652024-01-24T19:11:35.334Z09/30 Read block 228 len 32768
141662024-01-24T19:11:35.334Z10/30 Read block 15 len 20480
141672024-01-24T19:11:35.334Z11/30 Write block 549 len 36864 data: 10 10 11 10 9 9 7 9 11
141682024-01-24T19:11:35.335Z12/30 Read block 528 len 16384
141692024-01-24T19:11:35.335Z13/30 Read block 215 len 24576
141702024-01-24T19:11:35.335Z14/30 Read block 34 len 8192
141712024-01-24T19:11:35.335Z15/30 Write block 236 len 16384 data: 9 8 7 7
141722024-01-24T19:11:35.335Z16/30 Read block 414 len 28672
141732024-01-24T19:11:35.335Z17/30 Read block 571 len 8192
141742024-01-24T19:11:35.335Z18/30 Read block 242 len 28672
141752024-01-24T19:11:35.335Z19/30 Read block 564 len 12288
141762024-01-24T19:11:35.335Z20/30 Read block 422 len 32768
141772024-01-24T19:11:35.336Z21/30 Write block 567 len 16384 data: 13 15 14 9
141782024-01-24T19:11:35.336Z22/30 Write block 400 len 12288 data: 7 8 7
141792024-01-24T19:11:35.336Z23/30 Write block 566 len 32768 data: 10 14 16 15 10 10 12 9
141802024-01-24T19:11:35.336Z24/30 Read block 587 len 28672
141812024-01-24T19:11:35.347Z25/30 Write block 561 len 36864 data: 11 8 8 7 10 11 15 17 16
141822024-01-24T19:11:35.348Z26/30 Read block 560 len 32768
141832024-01-24T19:11:35.350Z27/30 Read block 372 len 20480
141842024-01-24T19:11:35.350Z28/30 Write block 384 len 8192 data: 7 5
141852024-01-24T19:11:35.350Z29/30 Write block 546 len 8192 data: 12 13
141862024-01-24T19:11:35.394Z30/30 Read block 95 len 8192
141872024-01-24T19:11:35.395Z----------------------------------------------------------------
141882024-01-24T19:11:35.395Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:28
141892024-01-24T19:11:35.395ZStates: Active Active Active
141902024-01-24T19:11:35.395ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
141912024-01-24T19:11:35.395Z 3 Acked 1002 Write 10 Done Done Done false
141922024-01-24T19:11:35.395Z 4 Acked 1003 Read 7 Done Done Done false
141932024-01-24T19:11:35.395Z 5 Acked 1004 Write 6 Done Sent Done false
141942024-01-24T19:11:35.395Z 6 Acked 1005 Read 9 Done Sent Done false
141952024-01-24T19:11:35.396Z 7 Acked 1006 Write 9 Done Sent Done false
141962024-01-24T19:11:35.396Z 8 Acked 1007 Read 5 Done Sent Done false
141972024-01-24T19:11:35.396Z 9 Acked 1008 Read 8 Done Sent Done false
141982024-01-24T19:11:35.396Z 10 Acked 1009 Read 5 Done Sent Done false
141992024-01-24T19:11:35.397Z 11 Acked 1010 Write 9 Done Sent Done false
142002024-01-24T19:11:35.397Z 12 Acked 1011 Read 4 Done Sent Done false
142012024-01-24T19:11:35.397Z 13 Acked 1012 Read 6 Done Sent Done false
142022024-01-24T19:11:35.397Z 14 Acked 1013 Read 2 Done Sent Done false
142032024-01-24T19:11:35.397Z 15 Acked 1014 Write 4 Done Sent Done false
142042024-01-24T19:11:35.397Z 16 Acked 1015 Read 7 Done Sent Done false
142052024-01-24T19:11:35.397Z 17 Acked 1016 Read 2 Done Sent Done false
142062024-01-24T19:11:35.397Z 18 Acked 1017 Read 7 Done Sent Done false
142072024-01-24T19:11:35.397Z 19 Acked 1018 Read 3 Done Sent Done false
142082024-01-24T19:11:35.397Z 20 Acked 1019 Read 8 Done Sent Done false
142092024-01-24T19:11:35.397Z 21 Acked 1020 Write 4 Done Sent Done false
142102024-01-24T19:11:35.398Z 22 Acked 1021 Write 3 Done Sent Done false
142112024-01-24T19:11:35.398Z 23 Acked 1022 Write 8 Done Sent Done false
142122024-01-24T19:11:35.398Z 24 Acked 1023 Read 7 Done Sent Done false
142132024-01-24T19:11:35.398Z 25 Acked 1024 Write 9 Done Sent Done false
142142024-01-24T19:11:35.398Z 26 Acked 1025 Read 8 Done Sent Done false
142152024-01-24T19:11:35.398Z 27 Acked 1026 Read 5 Done Sent Done false
142162024-01-24T19:11:35.398Z 28 Acked 1027 Write 2 Done Sent Done false
142172024-01-24T19:11:35.398Z 29 Acked 1028 Write 2 Done Sent Sent false
142182024-01-24T19:11:35.398Z 30 Acked 1029 Read 2 Done Sent Sent false
142192024-01-24T19:11:35.399Z STATES DS:0 DS:1 DS:2 TOTAL
142202024-01-24T19:11:35.399Z New 0 0 0 0
142212024-01-24T19:11:35.399Z Sent 0 26 2 28
142222024-01-24T19:11:35.399Z Done 28 2 26 56
142232024-01-24T19:11:35.399Z Skipped 0 0 0 0
142242024-01-24T19:11:35.399Z Error 0 0 0 0
142252024-01-24T19:11:35.399ZLast Flush: 1001 1001 1001
142262024-01-24T19:11:35.400ZDownstairs last five completed: 1001 1000
142272024-01-24T19:11:35.400ZUpstairs last five completed: 30 29 28 27 26
142282024-01-24T19:11:35.400ZWrote out file "/tmp/test_repair_verify.data"
142292024-01-24T19:11:35.400Z
142302024-01-24T19:11:35.400ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142312024-01-24T19:11:35.404ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142322024-01-24T19:11:35.404Z 0 000-019 126 126 126 215 215 215 F F F
142332024-01-24T19:11:35.405Z 1 020-039 128 128 128 221 221 221 F F F
142342024-01-24T19:11:35.405Z 2 040-059 128 128 128 221 221 221 F F F
142352024-01-24T19:11:35.405Z 3 060-079 125 125 125 213 213 213 F F F
142362024-01-24T19:11:35.405Z 4 080-099 114 114 114 196 196 196 T F T <---
142372024-01-24T19:11:35.405Z 5 100-119 132 132 132 228 228 228 T F T <---
142382024-01-24T19:11:35.405Z 6 120-139 119 119 119 203 203 203 F F F
142392024-01-24T19:11:35.405Z 7 140-159 131 131 131 227 227 227 F F F
142402024-01-24T19:11:35.405Z 8 160-179 122 122 122 208 208 208 F F F
142412024-01-24T19:11:35.405Z 9 180-199 130 130 130 223 223 223 T T T
142422024-01-24T19:11:35.405Z 10 200-219 129 129 129 222 222 222 F F F
142432024-01-24T19:11:35.405Z 11 220-239 130 130 130 226 226 226 T F T <---
142442024-01-24T19:11:35.405Z 12 240-259 130 130 130 225 225 225 F F F
142452024-01-24T19:11:35.406Z 13 260-279 122 122 122 207 207 207 F F F
142462024-01-24T19:11:35.406Z 14 280-299 129 129 129 222 222 222 F F F
142472024-01-24T19:11:35.406Z 15 300-319 130 130 130 226 226 226 F F F
142482024-01-24T19:11:35.406Z 16 320-339 124 124 124 212 212 212 F F F
142492024-01-24T19:11:35.406Z 17 340-359 130 130 130 223 223 223 F F F
142502024-01-24T19:11:35.406Z 18 360-379 126 126 126 215 215 215 F F F
142512024-01-24T19:11:35.406Z 19 380-399 123 123 123 209 209 209 T F T <---
142522024-01-24T19:11:35.406Z 20 400-419 116 116 116 200 200 200 T F T <---
142532024-01-24T19:11:35.407Z 21 420-439 126 126 126 214 214 214 F F F
142542024-01-24T19:11:35.407Z 22 440-459 120 120 120 204 204 204 F F F
142552024-01-24T19:11:35.407Z 23 460-479 129 129 129 222 222 222 F F F
142562024-01-24T19:11:35.407Z 24 480-499 130 130 130 225 225 225 F F F
142572024-01-24T19:11:35.407Z 25 500-519 126 126 126 217 217 217 F F F
142582024-01-24T19:11:35.407Z 26 520-539 129 129 129 222 222 222 F F F
142592024-01-24T19:11:35.407Z 27 540-559 129 129 129 222 222 222 T F T <---
142602024-01-24T19:11:35.407Z 28 560-579 130 130 130 224 224 224 T F T <---
142612024-01-24T19:11:35.407Z 29 580-599 131 131 131 227 227 227 F F F
142622024-01-24T19:11:35.407ZMax gen: 132, Max flush: 228
142632024-01-24T19:11:35.408ZError: Difference in extent metadata found!
142642024-01-24T19:11:35.408ZA Difference in extent metadata is expected here
142652024-01-24T19:11:35.408ZOn loop 66
142662024-01-24T19:11:35.408Z
142672024-01-24T19:11:35.408ZVerifying data now
142682024-01-24T19:11:38.808ZNo change after verify
142692024-01-24T19:11:38.808ZLoop: 66 Downstairs dump after verify (and repair):
142702024-01-24T19:11:38.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142712024-01-24T19:11:38.819Z 0 000-019 126 126 126 215 215 215 F F F
142722024-01-24T19:11:38.824Z 1 020-039 128 128 128 221 221 221 F F F
142732024-01-24T19:11:38.825Z 2 040-059 128 128 128 221 221 221 F F F
142742024-01-24T19:11:38.825Z 3 060-079 125 125 125 213 213 213 F F F
142752024-01-24T19:11:38.825Z 4 080-099 133 133 133 229 229 229 F F F
142762024-01-24T19:11:38.826Z 5 100-119 133 133 133 229 229 229 F F F
142772024-01-24T19:11:38.826Z 6 120-139 119 119 119 203 203 203 F F F
142782024-01-24T19:11:38.826Z 7 140-159 131 131 131 227 227 227 F F F
142792024-01-24T19:11:38.827Z 8 160-179 122 122 122 208 208 208 F F F
142802024-01-24T19:11:38.827Z 9 180-199 133 133 133 229 229 229 F F F
142812024-01-24T19:11:38.827Z 10 200-219 129 129 129 222 222 222 F F F
142822024-01-24T19:11:38.827Z 11 220-239 133 133 133 229 229 229 F F F
142832024-01-24T19:11:38.828Z 12 240-259 130 130 130 225 225 225 F F F
142842024-01-24T19:11:38.828Z 13 260-279 122 122 122 207 207 207 F F F
142852024-01-24T19:11:38.829Z 14 280-299 129 129 129 222 222 222 F F F
142862024-01-24T19:11:38.829Z 15 300-319 130 130 130 226 226 226 F F F
142872024-01-24T19:11:38.830Z 16 320-339 124 124 124 212 212 212 F F F
142882024-01-24T19:11:38.830Z 17 340-359 130 130 130 223 223 223 F F F
142892024-01-24T19:11:38.830Z 18 360-379 126 126 126 215 215 215 F F F
142902024-01-24T19:11:38.830Z 19 380-399 133 133 133 229 229 229 F F F
142912024-01-24T19:11:38.830Z 20 400-419 133 133 133 229 229 229 F F F
142922024-01-24T19:11:38.830Z 21 420-439 126 126 126 214 214 214 F F F
142932024-01-24T19:11:38.830Z 22 440-459 120 120 120 204 204 204 F F F
142942024-01-24T19:11:38.830Z 23 460-479 129 129 129 222 222 222 F F F
142952024-01-24T19:11:38.830Z 24 480-499 130 130 130 225 225 225 F F F
142962024-01-24T19:11:38.830Z 25 500-519 126 126 126 217 217 217 F F F
142972024-01-24T19:11:38.830Z 26 520-539 129 129 129 222 222 222 F F F
142982024-01-24T19:11:38.830Z 27 540-559 133 133 133 229 229 229 F F F
142992024-01-24T19:11:38.831Z 28 560-579 133 133 133 229 229 229 F F F
143002024-01-24T19:11:38.831Z 29 580-599 131 131 131 227 227 227 F F F
143012024-01-24T19:11:38.831ZMax gen: 133, Max flush: 229
143022024-01-24T19:11:38.831Z
143032024-01-24T19:11:38.831ZBegin loop 67
143042024-01-24T19:11:38.831ZDownstairs to restart: 0
143052024-01-24T19:11:38.868Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.524076346Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665}
143062024-01-24T19:11:38.896Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.524596289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","Crucible runtime is spawned
143072024-01-24T19:11:38.896Zpid":1665}
143082024-01-24T19:11:38.896Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.524864193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665}
143092024-01-24T19:11:38.896Z{"msg":"Crucible 8d4046a4-8969-43c2-aa2e-2148f61f102a has session id: 3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.524917347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143102024-01-24T19:11:38.898Z{"msg":"Upstairs opts: Upstairs UUID: 8d4046a4-8969-43c2-aa2e-2148f61f102a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525017558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143112024-01-24T19:11:38.898Z{"msg":"Crucible stats registered with UUID: 8d4046a4-8969-43c2-aa2e-2148f61f102a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525097457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143122024-01-24T19:11:38.898Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525139629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665}
143132024-01-24T19:11:38.898Z{"msg":"8d4046a4-8969-43c2-aa2e-2148f61f102a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.52517231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143142024-01-24T19:11:38.898Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525251485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143152024-01-24T19:11:38.898Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525300883Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143162024-01-24T19:11:38.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525348163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143172024-01-24T19:11:38.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525385029Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143182024-01-24T19:11:38.899Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.52546892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143192024-01-24T19:11:38.899Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525711484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143202024-01-24T19:11:38.899Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525872892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143212024-01-24T19:11:38.899Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.525922084Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143222024-01-24T19:11:38.900Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.526009072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143232024-01-24T19:11:38.900Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.526091209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143242024-01-24T19:11:38.900Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.526138308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143252024-01-24T19:11:38.900Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.5261776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143262024-01-24T19:11:38.900Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.526999217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143272024-01-24T19:11:38.902Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.527079544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143282024-01-24T19:11:38.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.527186736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143292024-01-24T19:11:38.903Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.527358856Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143302024-01-24T19:11:38.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.52739313Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143312024-01-24T19:11:38.904Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.527473867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143322024-01-24T19:11:38.904Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.528294331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143332024-01-24T19:11:38.904Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.528555274Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143342024-01-24T19:11:38.904Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.528633954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143352024-01-24T19:11:38.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529030251Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143362024-01-24T19:11:38.905Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529118819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143372024-01-24T19:11:38.905Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529149935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143382024-01-24T19:11:38.905Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529547071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143392024-01-24T19:11:38.905Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529658367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143402024-01-24T19:11:38.906Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529739964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143412024-01-24T19:11:38.906Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529817315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143422024-01-24T19:11:38.906Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529893361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143432024-01-24T19:11:38.907Z{"msg":"[0]R flush_numbers[0..12]: [215, 221, 221, 213, 229, 229, 203, 227, 208, 229, 222, 229]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.529969902Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143442024-01-24T19:11:38.907Z{"msg":"[0]R generation[0..12]: [126, 128, 128, 125, 133, 133, 119, 131, 122, 133, 129, 133]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530045823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143452024-01-24T19:11:38.907Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530120343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143462024-01-24T19:11:38.907Z{"msg":"[1]R flush_numbers[0..12]: [215, 221, 221, 213, 229, 229, 203, 227, 208, 229, 222, 229]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530248316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"Wait for a query_work_queue command to finish before sending IO
143472024-01-24T19:11:38.907Zdownstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143482024-01-24T19:11:38.907Z{"msg":"[1]R generation[0..12]: [126, 128, 128, 125, 133, 133, 119, 131, 122, 133, 129, 133]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530300749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143492024-01-24T19:11:38.907Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530373021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143502024-01-24T19:11:38.908Z{"msg":"[2]R flush_numbers[0..12]: [215, 221, 221, 213, 229, 229, 203, 227, 208, 229, 222, 229]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530448358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143512024-01-24T19:11:38.908Z{"msg":"[2]R generation[0..12]: [126, 128, 128, 125, 133, 133, 119, 131, 122, 133, 129, 133]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530521632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143522024-01-24T19:11:38.908Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530604989Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143532024-01-24T19:11:38.908Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530682663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143542024-01-24T19:11:38.908Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530763904Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143552024-01-24T19:11:38.908Z{"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530838736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143562024-01-24T19:11:38.908Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530910605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143572024-01-24T19:11:38.908ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143582024-01-24T19:11:38.909Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.530989153Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143592024-01-24T19:11:38.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531061224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143602024-01-24T19:11:38.910Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531113751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143612024-01-24T19:11:38.911Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531139725Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143622024-01-24T19:11:38.911Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531182456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143632024-01-24T19:11:38.911Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531206594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143642024-01-24T19:11:38.911Z{"msg":"8d4046a4-8969-43c2-aa2e-2148f61f102a is now active with session: 3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531248764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143652024-01-24T19:11:38.911Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531272627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665,"session_id":"3e1a39ca-bd1e-4fae-bf5f-3e7be2c0da13"}
143662024-01-24T19:11:38.911Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:37.531313935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1665}
143672024-01-24T19:11:38.912ZLoading write count information from file "/tmp/test_repair_verify.data"
143682024-01-24T19:11:38.912ZRun Repair workload
143692024-01-24T19:11:38.912Z01/30 Write block 48 len 20480 data: 12 13 15 12 11
143702024-01-24T19:11:38.912Z02/30 Read block 318 len 28672
143712024-01-24T19:11:38.912Z03/30 Read block 569 len 24576
143722024-01-24T19:11:38.912Z04/30 Flush
143732024-01-24T19:11:38.912Z05/30 Write block 109 len 16384 data: 13 12 11 10
143742024-01-24T19:11:38.913Z06/30 Write block 417 len 36864 data: 6 8 10 8 8 9 7 6 6
143752024-01-24T19:11:38.914Z07/30 Read block 144 len 28672
143762024-01-24T19:11:38.914Z08/30 Write block 265 len 4096 data: 10
143772024-01-24T19:11:38.914Z09/30 Read block 464 len 40960
143782024-01-24T19:11:38.924Z10/30 Read block 32 len 12288
143792024-01-24T19:11:38.924Z11/30 Read block 552 len 36864
143802024-01-24T19:11:38.924Z12/30 Read block 580 len 20480
143812024-01-24T19:11:38.924Z13/30 Read block 353 len 4096
143822024-01-24T19:11:38.924Z14/30 Read block 408 len 36864
143832024-01-24T19:11:38.925Z15/30 Write block 106 len 36864 data: 13 14 15 14 13 12 11 8 5
143842024-01-24T19:11:38.925Z16/30 Read block 409 len 8192
143852024-01-24T19:11:38.925Z17/30 Flush
143862024-01-24T19:11:38.925Z18/30 Flush
143872024-01-24T19:11:38.925Z19/30 Write block 542 len 32768 data: 13 12 14 12 13 14 11 11
143882024-01-24T19:11:38.925Z20/30 Write block 167 len 36864 data: 12 14 12 11 12 13 11 14 12
143892024-01-24T19:11:38.926Z21/30 Read block 315 len 12288
143902024-01-24T19:11:38.926Z22/30 Read block 389 len 20480
143912024-01-24T19:11:38.926Z23/30 Flush
143922024-01-24T19:11:38.926Z24/30 Flush
143932024-01-24T19:11:38.926Z25/30 Write block 130 len 4096 data: 10
143942024-01-24T19:11:38.926Z26/30 Flush
143952024-01-24T19:11:38.927Z27/30 Write block 51 len 24576 data: 13 12 11 10 10 11
143962024-01-24T19:11:38.927Z28/30 Read block 81 len 20480
143972024-01-24T19:11:38.927Z29/30 Write block 228 len 24576 data: 9 8 10 10 10 12
143982024-01-24T19:11:38.927Z30/30 Read block 106 len 32768
143992024-01-24T19:11:38.927Z----------------------------------------------------------------
144002024-01-24T19:11:38.927Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144012024-01-24T19:11:38.927ZStates: Active Active Active
144022024-01-24T19:11:38.927ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144032024-01-24T19:11:38.927Z 1 Acked 1000 Write 5 Done Done Done false
144042024-01-24T19:11:38.927Z 2 Acked 1001 Read 7 Done Done Done false
144052024-01-24T19:11:38.927Z 3 Acked 1002 Read 6 Sent Done Done false
144062024-01-24T19:11:38.928Z 4 Acked 1003 Flush 0 Sent Done Done false
144072024-01-24T19:11:38.928Z 5 Acked 1004 Write 4 Sent Done Done false
144082024-01-24T19:11:38.928Z 6 Acked 1005 Write 9 Sent Done Done false
144092024-01-24T19:11:38.928Z 7 Acked 1006 Read 7 Sent Done Done false
144102024-01-24T19:11:38.928Z 8 Acked 1007 Write 1 Sent Done Done false
144112024-01-24T19:11:38.928Z 9 Acked 1008 Read 10 Sent Done Done false
144122024-01-24T19:11:38.928Z 10 Acked 1009 Read 3 Sent Done Done false
144132024-01-24T19:11:38.928Z 11 Acked 1010 Read 9 Sent Done Done false
144142024-01-24T19:11:38.928Z 12 Acked 1011 Read 5 Sent Done Done false
144152024-01-24T19:11:38.928Z 13 Acked 1012 Read 1 Sent Done Done false
144162024-01-24T19:11:38.928Z 14 Acked 1013 Read 9 Sent Done Done false
144172024-01-24T19:11:38.928Z 15 Acked 1014 Write 9 Sent Done Done false
144182024-01-24T19:11:38.931Z 16 Acked 1015 Read 2 Sent Done Done false
144192024-01-24T19:11:38.931Z 17 Acked 1016 Flush 0 Sent Done Done false
144202024-01-24T19:11:38.931Z 18 Acked 1017 Flush 0 Sent Done Done false
144212024-01-24T19:11:38.931Z 19 Acked 1018 Write 8 Sent Done Done false
144222024-01-24T19:11:38.931Z 20 Acked 1019 Write 9 Sent Done Done false
144232024-01-24T19:11:38.932Z 21 Acked 1020 Read 3 Sent Done Done false
144242024-01-24T19:11:38.932Z 22 Acked 1021 Read 5 Sent Done Done false
144252024-01-24T19:11:38.932Z 23 Acked 1022 Flush 0 Sent Done Done false
144262024-01-24T19:11:38.932Z 24 Acked 1023 Flush 0 Sent Done Done false
144272024-01-24T19:11:38.932Z 25 Acked 1024 Write 1 Sent Done Done false
144282024-01-24T19:11:38.932Z 26 Acked 1025 Flush 0 Sent Done Done false
144292024-01-24T19:11:38.932Z 27 Acked 1026 Write 6 Sent Done Done false
144302024-01-24T19:11:38.932Z 28 Acked 1027 Read 5 Sent Done Done false
144312024-01-24T19:11:38.932Z 29 Acked 1028 Write 6 Sent Done Done false
144322024-01-24T19:11:38.933Z 30 Acked 1029 Read 8 Sent Done Sent false
144332024-01-24T19:11:38.933Z STATES DS:0 DS:1 DS:2 TOTAL
144342024-01-24T19:11:38.933Z New 0 0 0 0
144352024-01-24T19:11:38.933Z Sent 28 0 1 29
144362024-01-24T19:11:38.933Z Done 2 30 29 61
144372024-01-24T19:11:38.933Z Skipped 0 0 0 0
144382024-01-24T19:11:38.933Z Error 0 0 0 0
144392024-01-24T19:11:38.933ZLast Flush: 0 1025 1025
144402024-01-24T19:11:38.933ZDownstairs last five completed:
144412024-01-24T19:11:38.933ZUpstairs last five completed: 30 29 28 27 26
144422024-01-24T19:11:38.934ZWrote out file "/tmp/test_repair_verify.data"
144432024-01-24T19:11:38.963Z
144442024-01-24T19:11:38.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144452024-01-24T19:11:38.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144462024-01-24T19:11:38.982Z 0 000-019 126 126 126 215 215 215 F F F
144472024-01-24T19:11:38.982Z 1 020-039 128 128 128 221 221 221 F F F
144482024-01-24T19:11:38.982Z 2 040-059 128 134 134 221 230 230 T T T <---
144492024-01-24T19:11:38.982Z 3 060-079 125 125 125 213 213 213 F F F
144502024-01-24T19:11:38.982Z 4 080-099 133 133 133 229 229 229 F F F
144512024-01-24T19:11:38.982Z 5 100-119 133 134 134 229 231 231 F F F <---
144522024-01-24T19:11:38.983Z 6 120-139 119 134 134 203 235 235 F F F <---
144532024-01-24T19:11:38.983Z 7 140-159 131 131 131 227 227 227 F F F
144542024-01-24T19:11:38.983Z 8 160-179 122 134 134 208 233 233 F F F <---
144552024-01-24T19:11:38.983Z 9 180-199 133 133 133 229 229 229 F F F
144562024-01-24T19:11:38.983Z 10 200-219 129 129 129 222 222 222 F F F
144572024-01-24T19:11:38.984Z 11 220-239 133 133 133 229 229 229 F T T <---
144582024-01-24T19:11:38.984Z 12 240-259 130 130 130 225 225 225 F F F
144592024-01-24T19:11:38.984Z 13 260-279 122 134 134 207 231 231 F F F <---
144602024-01-24T19:11:38.984Z 14 280-299 129 129 129 222 222 222 F F F
144612024-01-24T19:11:38.984Z 15 300-319 130 130 130 226 226 226 F F F
144622024-01-24T19:11:38.985Z 16 320-339 124 124 124 212 212 212 F F F
144632024-01-24T19:11:38.987Z 17 340-359 130 130 130 223 223 223 F F F
144642024-01-24T19:11:38.987Z 18 360-379 126 126 126 215 215 215 F F F
144652024-01-24T19:11:38.987Z 19 380-399 133 133 133 229 229 229 F F F
144662024-01-24T19:11:38.987Z 20 400-419 133 134 134 229 231 231 F F F <---
144672024-01-24T19:11:38.987Z 21 420-439 126 134 134 214 231 231 F F F <---
144682024-01-24T19:11:38.987Z 22 440-459 120 120 120 204 204 204 F F F
144692024-01-24T19:11:38.987Z 23 460-479 129 129 129 222 222 222 F F F
144702024-01-24T19:11:38.987Z 24 480-499 130 130 130 225 225 225 F F F
144712024-01-24T19:11:38.987Z 25 500-519 126 126 126 217 217 217 F F F
144722024-01-24T19:11:38.987Z 26 520-539 129 129 129 222 222 222 F F F
144732024-01-24T19:11:38.988Z 27 540-559 133 134 134 229 233 233 F F F <---
144742024-01-24T19:11:38.988Z 28 560-579 133 133 133 229 229 229 F F F
144752024-01-24T19:11:38.988Z 29 580-599 131 131 131 227 227 227 F F F
144762024-01-24T19:11:38.988ZMax gen: 134, Max flush: 235
144772024-01-24T19:11:38.988ZError: Difference in extent metadata found!
144782024-01-24T19:11:38.988ZA Difference in extent metadata is expected here
144792024-01-24T19:11:38.988ZOn loop 67
144802024-01-24T19:11:38.988Z
144812024-01-24T19:11:38.988ZVerifying data now
144822024-01-24T19:11:43.813ZNo change after verify
144832024-01-24T19:11:43.813ZLoop: 67 Downstairs dump after verify (and repair):
144842024-01-24T19:11:44.254ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144852024-01-24T19:11:44.254Z 0 000-019 126 126 126 215 215 215 F F F
144862024-01-24T19:11:44.280Z 1 020-039 128 128 128 221 221 221 F F F
144872024-01-24T19:11:44.280Z 2 040-059 135 135 135 236 236 236 F F F
144882024-01-24T19:11:44.280Z 3 060-079 125 125 125 213 213 213 F F F
144892024-01-24T19:11:44.304Z 4 080-099 133 133 133 229 229 229 F F F
144902024-01-24T19:11:44.305Z 5 100-119 134 134 134 231 231 231 F F F
144912024-01-24T19:11:44.305Z 6 120-139 134 134 134 235 235 235 F F F
144922024-01-24T19:11:44.305Z 7 140-159 131 131 131 227 227 227 F F F
144932024-01-24T19:11:44.305Z 8 160-179 134 134 134 233 233 233 F F F
144942024-01-24T19:11:44.305Z 9 180-199 133 133 133 229 229 229 F F F
144952024-01-24T19:11:44.305Z 10 200-219 129 129 129 222 222 222 F F F
144962024-01-24T19:11:44.305Z 11 220-239 135 135 135 236 236 236 F F F
144972024-01-24T19:11:44.305Z 12 240-259 130 130 130 225 225 225 F F F
144982024-01-24T19:11:44.305Z 13 260-279 134 134 134 231 231 231 F F F
144992024-01-24T19:11:44.305Z 14 280-299 129 129 129 222 222 222 F F F
145002024-01-24T19:11:44.305Z 15 300-319 130 130 130 226 226 226 F F F
145012024-01-24T19:11:44.305Z 16 320-339 124 124 124 212 212 212 F F F
145022024-01-24T19:11:44.306Z 17 340-359 130 130 130 223 223 223 F F F
145032024-01-24T19:11:44.306Z 18 360-379 126 126 126 215 215 215 F F F
145042024-01-24T19:11:44.306Z 19 380-399 133 133 133 229 229 229 F F F
145052024-01-24T19:11:44.306Z 20 400-419 134 134 134 231 231 231 F F F
145062024-01-24T19:11:44.306Z 21 420-439 134 134 134 231 231 231 F F F
145072024-01-24T19:11:44.306Z 22 440-459 120 120 120 204 204 204 F F F
145082024-01-24T19:11:44.306Z 23 460-479 129 129 129 222 222 222 F F F
145092024-01-24T19:11:44.306Z 24 480-499 130 130 130 225 225 225 F F F
145102024-01-24T19:11:44.306Z 25 500-519 126 126 126 217 217 217 F F F
145112024-01-24T19:11:44.306Z 26 520-539 129 129 129 222 222 222 F F F
145122024-01-24T19:11:44.306Z 27 540-559 134 134 134 233 233 233 F F F
145132024-01-24T19:11:44.307Z 28 560-579 133 133 133 229 229 229 F F F
145142024-01-24T19:11:44.307Z 29 580-599 131 131 131 227 227 227 F F F
145152024-01-24T19:11:44.307ZMax gen: 135, Max flush: 236
145162024-01-24T19:11:44.307Z
145172024-01-24T19:11:44.307ZBegin loop 68
145182024-01-24T19:11:44.307ZDownstairs to restart: 2
145192024-01-24T19:11:44.308Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.541365211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673}
145202024-01-24T19:11:44.308Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.541847099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673}
145212024-01-24T19:11:44.308Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.541936536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673}
145222024-01-24T19:11:44.308Z{"msg":"Crucible 56eba4af-2f30-4d04-9095-579359ad1adf has session id: 69d67c5f-d2e4-4c4e-84c3-3b368dd3480f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.541963056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145232024-01-24T19:11:44.308Z{"msg":"Upstairs opts: Upstairs UUID: 56eba4af-2f30-4d04-9095-579359ad1adf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.542067065Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145242024-01-24T19:11:44.308Z{"msg":"Crucible stats registered with UUID: 56eba4af-2f30-4d04-9095-579359ad1adf","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.542099018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145252024-01-24T19:11:44.308ZCrucible runtime is spawned
145262024-01-24T19:11:44.308Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.542538931Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673}
145272024-01-24T19:11:44.309Z{"msg":"56eba4af-2f30-4d04-9095-579359ad1adf active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.54278103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145282024-01-24T19:11:44.309Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.542949596Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145292024-01-24T19:11:44.309Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543039566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145302024-01-24T19:11:44.309Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543099843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145312024-01-24T19:11:44.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543126558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145322024-01-24T19:11:44.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543267825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145332024-01-24T19:11:44.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543318592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145342024-01-24T19:11:44.310Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543404964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145352024-01-24T19:11:44.310Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543636468Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145362024-01-24T19:11:44.310Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543803475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145372024-01-24T19:11:44.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.543968604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145382024-01-24T19:11:44.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544106966Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145392024-01-24T19:11:44.310Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544153458Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145402024-01-24T19:11:44.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544647818Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145412024-01-24T19:11:44.310Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544685471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145422024-01-24T19:11:44.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544842799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145432024-01-24T19:11:44.310Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.544892409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145442024-01-24T19:11:44.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.545138555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145452024-01-24T19:11:44.310Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.545230438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145462024-01-24T19:11:44.310Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.545818088Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145472024-01-24T19:11:44.310Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.54609587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145482024-01-24T19:11:44.310Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546500841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145492024-01-24T19:11:44.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546727781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145502024-01-24T19:11:44.310Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546792538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145512024-01-24T19:11:44.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546821505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145522024-01-24T19:11:44.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.54686518Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145532024-01-24T19:11:44.310Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546891562Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145542024-01-24T19:11:44.311Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.546934817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145552024-01-24T19:11:44.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547251897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145562024-01-24T19:11:44.311Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547339799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145572024-01-24T19:11:44.312Z{"msg":"[0]R flush_numbers[0..12]: [215, 221, 236, 213, 229, 231, 235, 227, 233, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547369453Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145582024-01-24T19:11:44.312Z{"msg":"[0]R generation[0..12]: [126, 128, 135, 125, 133, 134, 134, 131, 134, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547393854Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145592024-01-24T19:11:44.312Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547418588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145602024-01-24T19:11:44.312Z{"msg":"[1]R flush_numbers[0..12]: [215, 221, 236, 213, 229, 231, 235, 227, 233, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547465492Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145612024-01-24T19:11:44.312Z{"msg":"[1]R generation[0..12]: [126, 128, 135, 125, 133, 134, 134, 131, 134, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547493046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145622024-01-24T19:11:44.312Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547546662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145632024-01-24T19:11:44.312Z{"msg":"[2]R flush_numbers[0..12]: [215, 221, 236, 213, 229, 231, 235, 227, 233, 229, 222, 236]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
145642024-01-24T19:11:44.312Z,"time":"2024-01-24T19:11:42.547573861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145652024-01-24T19:11:44.312Z{"msg":"[2]R generation[0..12]: [126, 128, 135, 125, 133, 134, 134, 131, 134, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547694433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145662024-01-24T19:11:44.312Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547756998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145672024-01-24T19:11:44.314Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547783309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145682024-01-24T19:11:44.314Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547835391Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145692024-01-24T19:11:44.314Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547860446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145702024-01-24T19:11:44.314Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547907784Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145712024-01-24T19:11:44.314Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547933588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145722024-01-24T19:11:44.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547954279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145732024-01-24T19:11:44.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.547997432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145742024-01-24T19:11:44.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548023198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145752024-01-24T19:11:44.314Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548066225Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145762024-01-24T19:11:44.314Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548090648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145772024-01-24T19:11:44.314Z{"msg":"56eba4af-2f30-4d04-9095-579359ad1adf is now active with session: 69d67c5f-d2e4-4c4e-84c3-3b368dd3480f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548138166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145782024-01-24T19:11:44.314Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548161678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673,"session_id":"69d67c5f-d2e4-4c4e-84c3-3b368dd3480f"}
145792024-01-24T19:11:44.314Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:42.548182996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1673}
145802024-01-24T19:11:44.314ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145812024-01-24T19:11:44.314ZLoading write count information from file "/tmp/test_repair_verify.data"
145822024-01-24T19:11:44.314ZRun Repair workload
145832024-01-24T19:11:44.314Z01/30 Write block 67 len 8192 data: 6 4
145842024-01-24T19:11:44.314Z02/30 Write block 109 len 36864 data: 15 14 13 12 9 6 6 6 5
145852024-01-24T19:11:44.314Z03/30 Read block 457 len 20480
145862024-01-24T19:11:44.326Z04/30 Read block 375 len 20480
145872024-01-24T19:11:44.327Z05/30 Write block 369 len 28672 data: 11 11 10 9 10 10 13
145882024-01-24T19:11:44.327Z06/30 Read block 521 len 28672
145892024-01-24T19:11:44.327Z07/30 Flush
145902024-01-24T19:11:44.327Z08/30 Flush
145912024-01-24T19:11:44.327Z09/30 Write block 317 len 4096 data: 13
145922024-01-24T19:11:44.327Z10/30 Write block 121 len 12288 data: 10 10 9
145932024-01-24T19:11:44.327Z11/30 Write block 19 len 24576 data: 8 8 10 10 9 11
145942024-01-24T19:11:44.327Z12/30 Flush
145952024-01-24T19:11:44.327Z13/30 Write block 395 len 36864 data: 11 12 10 8 7 8 9 8 5
145962024-01-24T19:11:44.327Z14/30 Read block 20 len 28672
145972024-01-24T19:11:44.327Z15/30 Write block 493 len 24576 data: 10 11 10 14 14 15
145982024-01-24T19:11:44.327Z16/30 Write block 408 len 32768 data: 11 6 9 9 8 8 5 5
145992024-01-24T19:11:44.327Z17/30 Read block 191 len 36864
146002024-01-24T19:11:44.327Z18/30 Write block 158 len 28672 data: 14 9 8 16 15 16 12
146012024-01-24T19:11:44.327Z19/30 Write block 344 len 28672 data: 11 11 9 9 7 10 8
146022024-01-24T19:11:44.327Z20/30 Read block 345 len 12288
146032024-01-24T19:11:44.327Z21/30 Write block 309 len 12288 data: 6 7 7
146042024-01-24T19:11:44.327Z22/30 Read block 361 len 24576
146052024-01-24T19:11:44.327Z23/30 Read block 348 len 32768
146062024-01-24T19:11:44.327Z24/30 Flush
146072024-01-24T19:11:44.328Z25/30 Write block 260 len 36864 data: 8 8 8 7 10 11 9 10 10
146082024-01-24T19:11:44.328Z26/30 Read block 285 len 20480
146092024-01-24T19:11:44.328Z27/30 Read block 21 len 8192
146102024-01-24T19:11:44.328Z28/30 Read block 134 len 4096
146112024-01-24T19:11:44.328Z29/30 Read block 468 len 32768
146122024-01-24T19:11:44.328Z30/30 Write block 435 len 28672 data: 11 10 13 10 10 9 6
146132024-01-24T19:11:44.328Z----------------------------------------------------------------
146142024-01-24T19:11:44.328Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
146152024-01-24T19:11:44.329ZStates: Active Active Active
146162024-01-24T19:11:44.329ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146172024-01-24T19:11:44.329Z 1 Acked 1000 Write 2 Done Done Done false
146182024-01-24T19:11:44.329Z 2 Acked 1001 Write 9 Done Done Sent false
146192024-01-24T19:11:44.329Z 3 Acked 1002 Read 5 Done Done Sent false
146202024-01-24T19:11:44.329Z 4 Acked 1003 Read 5 Done Done Sent false
146212024-01-24T19:11:44.329Z 5 Acked 1004 Write 7 Done Done Sent false
146222024-01-24T19:11:44.330Z 6 Acked 1005 Read 7 Done Done Sent false
146232024-01-24T19:11:44.330Z 7 Acked 1006 Flush 0 Done Done Sent false
146242024-01-24T19:11:44.330Z 8 Acked 1007 Flush 0 Done Done Sent false
146252024-01-24T19:11:44.330Z 9 Acked 1008 Write 1 Done Done Sent false
146262024-01-24T19:11:44.330Z 10 Acked 1009 Write 3 Done Done Sent false
146272024-01-24T19:11:44.330Z 11 Acked 1010 Write 6 Done Done Sent false
146282024-01-24T19:11:44.330Z 12 Acked 1011 Flush 0 Done Done Sent false
146292024-01-24T19:11:44.330Z 13 Acked 1012 Write 9 Done Done Sent false
146302024-01-24T19:11:44.330Z 14 Acked 1013 Read 7 Done Done Sent false
146312024-01-24T19:11:44.330Z 15 Acked 1014 Write 6 Done Done Sent false
146322024-01-24T19:11:44.330Z 16 Acked 1015 Write 8 Done Done Sent false
146332024-01-24T19:11:44.330Z 17 Acked 1016 Read 9 Done Done Sent false
146342024-01-24T19:11:44.330Z 18 Acked 1017 Write 7 Done Done Sent false
146352024-01-24T19:11:44.330Z 19 Acked 1018 Write 7 Done Done Sent false
146362024-01-24T19:11:44.330Z 20 Acked 1019 Read 3 Done Done Sent false
146372024-01-24T19:11:44.330Z 21 Acked 1020 Write 3 Done Done Sent false
146382024-01-24T19:11:44.330Z 22 Acked 1021 Read 6 Done Done Sent false
146392024-01-24T19:11:44.330Z 23 Acked 1022 Read 8 Done Done Sent false
146402024-01-24T19:11:44.330Z 24 Acked 1023 Flush 0 Done Done Sent false
146412024-01-24T19:11:44.330Z 25 Acked 1024 Write 9 Done Done Sent false
146422024-01-24T19:11:44.330Z 26 Acked 1025 Read 5 Done Done Sent false
146432024-01-24T19:11:44.331Z 27 Acked 1026 Read 2 Done Done Sent false
146442024-01-24T19:11:44.331Z 28 Acked 1027 Read 1 Done Done Sent false
146452024-01-24T19:11:44.331Z 29 Acked 1028 Read 8 Done Done Sent false
146462024-01-24T19:11:44.331Z 30 Acked 1029 Write 7 Sent Sent Sent false
146472024-01-24T19:11:44.331Z STATES DS:0 DS:1 DS:2 TOTAL
146482024-01-24T19:11:44.331Z New 0 0 0 0
146492024-01-24T19:11:44.331Z Sent 1 1 29 31
146502024-01-24T19:11:44.331Z Done 29 29 1 59
146512024-01-24T19:11:44.331Z Skipped 0 0 0 0
146522024-01-24T19:11:44.331Z Error 0 0 0 0
146532024-01-24T19:11:44.331ZLast Flush: 1023 1023 0
146542024-01-24T19:11:44.331ZDownstairs last five completed:
146552024-01-24T19:11:44.331ZUpstairs last five completed: 30 29 28 27 26
146562024-01-24T19:11:44.331ZWrote out file "/tmp/test_repair_verify.data"
146572024-01-24T19:11:44.331Z
146582024-01-24T19:11:44.331ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146592024-01-24T19:11:44.331ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146602024-01-24T19:11:44.331Z 0 000-019 136 136 126 239 239 215 F F F <---
146612024-01-24T19:11:44.331Z 1 020-039 136 136 128 239 239 221 F F F <---
146622024-01-24T19:11:44.331Z 2 040-059 135 135 135 236 236 236 F F F
146632024-01-24T19:11:44.331Z 3 060-079 136 136 125 237 237 213 F F T <---
146642024-01-24T19:11:44.331Z 4 080-099 133 133 133 229 229 229 F F F
146652024-01-24T19:11:44.332Z 5 100-119 136 136 134 237 237 231 F F F <---
146662024-01-24T19:11:44.332Z 6 120-139 136 136 134 239 239 235 F F F <---
146672024-01-24T19:11:44.332Z 7 140-159 136 136 131 240 240 227 F F F <---
146682024-01-24T19:11:44.332Z 8 160-179 136 136 134 240 240 233 F F F <---
146692024-01-24T19:11:44.332Z 9 180-199 133 133 133 229 229 229 F F F
146702024-01-24T19:11:44.333Z 10 200-219 129 129 129 222 222 222 F F F
146712024-01-24T19:11:44.333Z 11 220-239 135 135 135 236 236 236 F F F
146722024-01-24T19:11:44.333Z 12 240-259 130 130 130 225 225 225 F F F
146732024-01-24T19:11:44.333Z 13 260-279 134 134 134 231 231 231 T T F <---
146742024-01-24T19:11:44.333Z 14 280-299 129 129 129 222 222 222 F F F
146752024-01-24T19:11:44.333Z 15 300-319 136 136 130 240 240 226 F F F <---
146762024-01-24T19:11:44.333Z 16 320-339 124 124 124 212 212 212 F F F
146772024-01-24T19:11:44.333Z 17 340-359 136 136 130 240 240 223 F F F <---
146782024-01-24T19:11:44.333Z 18 360-379 136 136 126 237 237 215 F F F <---
146792024-01-24T19:11:44.333Z 19 380-399 136 136 133 240 240 229 F F F <---
146802024-01-24T19:11:44.333Z 20 400-419 136 136 134 240 240 231 F F F <---
146812024-01-24T19:11:44.333Z 21 420-439 134 134 134 231 231 231 T T F <---
146822024-01-24T19:11:44.333Z 22 440-459 120 120 120 204 204 204 T T F <---
146832024-01-24T19:11:44.333Z 23 460-479 129 129 129 222 222 222 F F F
146842024-01-24T19:11:44.333Z 24 480-499 136 136 130 240 240 225 F F F <---
146852024-01-24T19:11:44.333Z 25 500-519 126 126 126 217 217 217 F F F
146862024-01-24T19:11:44.346Z 26 520-539 129 129 129 222 222 222 F F F
146872024-01-24T19:11:44.346Z 27 540-559 134 134 134 233 233 233 F F F
146882024-01-24T19:11:44.346Z 28 560-579 133 133 133 229 229 229 F F F
146892024-01-24T19:11:44.346Z 29 580-599 131 131 131 227 227 227 F F F
146902024-01-24T19:11:44.346ZMax gen: 136, Max flush: 240
146912024-01-24T19:11:44.346ZError: Difference in extent metadata found!
146922024-01-24T19:11:44.346ZA Difference in extent metadata is expected here
146932024-01-24T19:11:44.346ZOn loop 68
146942024-01-24T19:11:44.347Z
146952024-01-24T19:11:44.349ZVerifying data now
146962024-01-24T19:11:51.153ZNo change after verify
146972024-01-24T19:11:51.153ZLoop: 68 Downstairs dump after verify (and repair):
146982024-01-24T19:11:51.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146992024-01-24T19:11:51.153Z 0 000-019 136 136 136 239 239 239 F F F
147002024-01-24T19:11:51.154Z 1 020-039 136 136 136 239 239 239 F F F
147012024-01-24T19:11:51.154Z 2 040-059 135 135 135 236 236 236 F F F
147022024-01-24T19:11:51.154Z 3 060-079 136 136 136 237 237 237 F F F
147032024-01-24T19:11:51.154Z 4 080-099 133 133 133 229 229 229 F F F
147042024-01-24T19:11:51.154Z 5 100-119 136 136 136 237 237 237 F F F
147052024-01-24T19:11:51.154Z 6 120-139 136 136 136 239 239 239 F F F
147062024-01-24T19:11:51.154Z 7 140-159 136 136 136 240 240 240 F F F
147072024-01-24T19:11:51.154Z 8 160-179 136 136 136 240 240 240 F F F
147082024-01-24T19:11:51.154Z 9 180-199 133 133 133 229 229 229 F F F
147092024-01-24T19:11:51.154Z 10 200-219 129 129 129 222 222 222 F F F
147102024-01-24T19:11:51.154Z 11 220-239 135 135 135 236 236 236 F F F
147112024-01-24T19:11:51.155Z 12 240-259 130 130 130 225 225 225 F F F
147122024-01-24T19:11:51.194Z 13 260-279 137 137 137 241 241 241 F F F
147132024-01-24T19:11:51.194Z 14 280-299 129 129 129 222 222 222 F F F
147142024-01-24T19:11:51.194Z 15 300-319 136 136 136 240 240 240 F F F
147152024-01-24T19:11:51.194Z 16 320-339 124 124 124 212 212 212 F F F
147162024-01-24T19:11:51.194Z 17 340-359 136 136 136 240 240 240 F F F
147172024-01-24T19:11:51.194Z 18 360-379 136 136 136 237 237 237 F F F
147182024-01-24T19:11:51.194Z 19 380-399 136 136 136 240 240 240 F F F
147192024-01-24T19:11:51.194Z 20 400-419 136 136 136 240 240 240 F F F
147202024-01-24T19:11:51.202Z 21 420-439 137 137 137 241 241 241 F F F
147212024-01-24T19:11:51.202Z 22 440-459 137 137 137 241 241 241 F F F
147222024-01-24T19:11:51.202Z 23 460-479 129 129 129 222 222 222 F F F
147232024-01-24T19:11:51.202Z 24 480-499 136 136 136 240 240 240 F F F
147242024-01-24T19:11:51.202Z 25 500-519 126 126 126 217 217 217 F F F
147252024-01-24T19:11:51.202Z 26 520-539 129 129 129 222 222 222 F F F
147262024-01-24T19:11:51.203Z 27 540-559 134 134 134 233 233 233 F F F
147272024-01-24T19:11:51.203Z 28 560-579 133 133 133 229 229 229 F F F
147282024-01-24T19:11:51.203Z 29 580-599 131 131 131 227 227 227 F F F
147292024-01-24T19:11:51.203ZMax gen: 137, Max flush: 241
147302024-01-24T19:11:51.203Z
147312024-01-24T19:11:51.203ZBegin loop 69
147322024-01-24T19:11:51.203ZDownstairs to restart: 0
147332024-01-24T19:11:51.203Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.058351434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681}
147342024-01-24T19:11:51.203Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.058848654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681}
147352024-01-24T19:11:51.203Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.058938195Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681}
147362024-01-24T19:11:51.203Z{"msg":"Crucible acbffc93-fdf4-42f4-9e55-1c3ecba91aa2 has session id: bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.058979156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147372024-01-24T19:11:51.203ZCrucible runtime is spawned
147382024-01-24T19:11:51.203Z{"msg":"Upstairs opts: Upstairs UUID: acbffc93-fdf4-42f4-9e55-1c3ecba91aa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059095144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147392024-01-24T19:11:51.206Z{"msg":"Crucible stats registered with UUID: acbffc93-fdf4-42f4-9e55-1c3ecba91aa2","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059192634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147402024-01-24T19:11:51.206Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059262772Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681}
147412024-01-24T19:11:51.206Z{"msg":"acbffc93-fdf4-42f4-9e55-1c3ecba91aa2 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059455743Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147422024-01-24T19:11:51.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059520797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147432024-01-24T19:11:51.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.05957663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147442024-01-24T19:11:51.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059627524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147452024-01-24T19:11:51.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059688912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147462024-01-24T19:11:51.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059773738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147472024-01-24T19:11:51.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059886127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147482024-01-24T19:11:51.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.059953674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147492024-01-24T19:11:51.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.060091888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147502024-01-24T19:11:51.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.060241286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147512024-01-24T19:11:51.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.060347494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147522024-01-24T19:11:51.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.060433536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147532024-01-24T19:11:51.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.060507228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147542024-01-24T19:11:51.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.061440291Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147552024-01-24T19:11:51.207Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06151113Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147562024-01-24T19:11:51.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06156122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147572024-01-24T19:11:51.207Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.061638127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147582024-01-24T19:11:51.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.062280456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147592024-01-24T19:11:51.208Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.062434411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147602024-01-24T19:11:51.209Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06253578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147612024-01-24T19:11:51.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.062930754Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147622024-01-24T19:11:51.209Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06332205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147632024-01-24T19:11:51.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.063604331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147642024-01-24T19:11:51.209Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.063696149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147652024-01-24T19:11:51.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06377848Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147662024-01-24T19:11:51.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.063890208Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147672024-01-24T19:11:51.209Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.063955002Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147682024-01-24T19:11:51.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064016946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147692024-01-24T19:11:51.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064077171Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147702024-01-24T19:11:51.210Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064148642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147712024-01-24T19:11:51.210Z{"msg":"[0]R flush_numbers[0..12]: [239, 239, 236, 237, 229, 237, 239, 240, 240, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064208758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147722024-01-24T19:11:51.210Z{"msg":"[0]R generation[0..12]: [136, 136, 135, 136, 133, 136, 136, 136, 136, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064271268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147732024-01-24T19:11:51.210Z{"msg"Wait for a query_work_queue command to finish before sending IO
147742024-01-24T19:11:51.210Z:"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064341204Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147752024-01-24T19:11:51.210Z{"msg":"[1]R flush_numbers[0..12]: [239, 239, 236, 237, 229, 237, 239, 240, 240, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064404273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147762024-01-24T19:11:51.210Z{"msg":"[1]R generation[0..12]: [136, 136, 135, 136, 133, 136, 136, 136, 136, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064466723Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147772024-01-24T19:11:51.210Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064533716Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147782024-01-24T19:11:51.210Z{"msg":"[2]R flush_numbers[0..12]: [239, 239, 236, 237, 229, 237, 239, 240, 240, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.06460441Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147792024-01-24T19:11:51.211Z{"msg":"[2]R generation[0..12]: [136, 136, 135, 136, 133, 136, 136, 136, 136, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064674318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147802024-01-24T19:11:51.211Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064748621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147812024-01-24T19:11:51.211Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064815953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147822024-01-24T19:11:51.211Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064886949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147832024-01-24T19:11:51.211Z{"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.064947349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147842024-01-24T19:11:51.211Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065007976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147852024-01-24T19:11:51.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065068864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147862024-01-24T19:11:51.211Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147872024-01-24T19:11:51.212Z"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065133801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147882024-01-24T19:11:51.212Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065192776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147892024-01-24T19:11:51.212Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065251851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147902024-01-24T19:11:51.212Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065312374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147912024-01-24T19:11:51.212Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065370967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147922024-01-24T19:11:51.212Z{"msg":"acbffc93-fdf4-42f4-9e55-1c3ecba91aa2 is now active with session: bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065430773Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147932024-01-24T19:11:51.212Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065487336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681,"session_id":"bdb16b59-a475-4d5e-a1c2-cee45ed2dfd0"}
147942024-01-24T19:11:51.212Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:48.065544908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1681}
147952024-01-24T19:11:51.212ZLoading write count information from file "/tmp/test_repair_verify.data"
147962024-01-24T19:11:51.212ZRun Repair workload
147972024-01-24T19:11:51.212ZError: Difference in extent metadata found!
147982024-01-24T19:11:51.212Z01/30 Write block 277 len 28672 data: 12 13 15 14 14 15 14
147992024-01-24T19:11:51.212Z02/30 Write block 167 len 16384 data: 13 15 13 12
148002024-01-24T19:11:51.212Z03/30 Read block 65 len 20480
148012024-01-24T19:11:51.212Z04/30 Read block 63 len 36864
148022024-01-24T19:11:51.212Z05/30 Read block 198 len 4096
148032024-01-24T19:11:51.212Z06/30 Flush
148042024-01-24T19:11:51.212Z07/30 Write block 285 len 32768 data: 10 9 10 12 11 9 8 9
148052024-01-24T19:11:51.222Z08/30 Write block 327 len 32768 data: 10 12 13 11 11 13 11 10
148062024-01-24T19:11:51.222Z09/30 Read block 374 len 32768
148072024-01-24T19:11:51.225Z10/30 Read block 430 len 16384
148082024-01-24T19:11:51.225Z11/30 Read block 80 len 8192
148092024-01-24T19:11:51.225Z12/30 Read block 237 len 8192
148102024-01-24T19:11:51.225Z13/30 Write block 381 len 20480 data: 11 9 10 8 6
148112024-01-24T19:11:51.225Z14/30 Flush
148122024-01-24T19:11:51.251Z15/30 Write block 0 len 32768 data: 3 5 6 7 7 7 9 10
148132024-01-24T19:11:51.252Z16/30 Write block 273 len 8192 data: 14 12
148142024-01-24T19:11:51.252Z17/30 Read block 133 len 4096
148152024-01-24T19:11:51.252Z18/30 Read block 330 len 36864
148162024-01-24T19:11:51.252Z19/30 Read block 215 len 24576
148172024-01-24T19:11:51.252Z20/30 Read block 17 len 24576
148182024-01-24T19:11:51.252Z21/30 Write block 585 len 12288 data: 9 10 10
148192024-01-24T19:11:51.252Z22/30 Read block 174 len 12288
148202024-01-24T19:11:51.252Z23/30 Write block 165 len 12288 data: 13 11 14
148212024-01-24T19:11:51.252Z24/30 Read block 356 len 12288
148222024-01-24T19:11:51.252Z25/30 Read block 21 len 20480
148232024-01-24T19:11:51.252Z26/30 Write block 497 len 8192 data: 15 16
148242024-01-24T19:11:51.252Z27/30 Write block 27 len 40960 data: 14 16 15 13 15 13 13 13 8 7
148252024-01-24T19:11:51.252Z28/30 Write block 51 len 28672 data: 14 13 12 11 11 12 11
148262024-01-24T19:11:51.252Z29/30 Read block 183 len 16384
148272024-01-24T19:11:51.252Z30/30 Read block 329 len 40960
148282024-01-24T19:11:51.252Z----------------------------------------------------------------
148292024-01-24T19:11:51.252Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
148302024-01-24T19:11:51.253ZStates: Active Active Active
148312024-01-24T19:11:51.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148322024-01-24T19:11:51.253Z 1 Acked 1000 Write 7 Done Done Done false
148332024-01-24T19:11:51.253Z 2 Acked 1001 Write 4 Done Done Done false
148342024-01-24T19:11:51.253Z 3 Acked 1002 Read 5 Done Done Done false
148352024-01-24T19:11:51.253Z 4 Acked 1003 Read 9 Sent Done Done false
148362024-01-24T19:11:51.253Z 5 Acked 1004 Read 1 Sent Done Done false
148372024-01-24T19:11:51.253Z 6 Acked 1005 Flush 0 Sent Done Done false
148382024-01-24T19:11:51.253Z 7 Acked 1006 Write 8 Sent Done Done false
148392024-01-24T19:11:51.253Z 8 Acked 1007 Write 8 Sent Done Done false
148402024-01-24T19:11:51.253Z 9 Acked 1008 Read 8 Sent Done Done false
148412024-01-24T19:11:51.253Z 10 Acked 1009 Read 4 Sent Done Done false
148422024-01-24T19:11:51.253Z 11 Acked 1010 Read 2 Sent Done Done false
148432024-01-24T19:11:51.253Z 12 Acked 1011 Read 2 Sent Done Done false
148442024-01-24T19:11:51.253Z 13 Acked 1012 Write 5 Sent Done Done false
148452024-01-24T19:11:51.254Z 14 Acked 1013 Flush 0 Sent Done Done false
148462024-01-24T19:11:51.254Z 15 Acked 1014 Write 8 Sent Done Done false
148472024-01-24T19:11:51.254Z 16 Acked 1015 Write 2 Sent Done Done false
148482024-01-24T19:11:51.254Z 17 Acked 1016 Read 1 Sent Done Done false
148492024-01-24T19:11:51.254Z 18 Acked 1017 Read 9 Sent Done Done false
148502024-01-24T19:11:51.254Z 19 Acked 1018 Read 6 Sent Done Done false
148512024-01-24T19:11:51.255Z 20 Acked 1019 Read 6 Sent Done Done false
148522024-01-24T19:11:51.255Z 21 Acked 1020 Write 3 Sent Done Done false
148532024-01-24T19:11:51.255Z 22 Acked 1021 Read 3 Sent Done Done false
148542024-01-24T19:11:51.255Z 23 Acked 1022 Write 3 Sent Done Done false
148552024-01-24T19:11:51.255Z 24 Acked 1023 Read 3 Sent Done Done false
148562024-01-24T19:11:51.255Z 25 Acked 1024 Read 5 Sent Done Done false
148572024-01-24T19:11:51.255Z 26 Acked 1025 Write 2 Sent Done Done false
148582024-01-24T19:11:51.255Z 27 Acked 1026 Write 10 Sent Done Done false
148592024-01-24T19:11:51.255Z 28 Acked 1027 Write 7 Sent Done Done false
148602024-01-24T19:11:51.255Z 29 Acked 1028 Read 4 Sent Done Done false
148612024-01-24T19:11:51.255Z 30 Acked 1029 Read 10 Sent Done Sent false
148622024-01-24T19:11:51.255Z STATES DS:0 DS:1 DS:2 TOTAL
148632024-01-24T19:11:51.255Z New 0 0 0 0
148642024-01-24T19:11:51.255Z Sent 27 0 1 28
148652024-01-24T19:11:51.255Z Done 3 30 29 62
148662024-01-24T19:11:51.255Z Skipped 0 0 0 0
148672024-01-24T19:11:51.255Z Error 0 0 0 0
148682024-01-24T19:11:51.255ZLast Flush: 0 1013 1013
148692024-01-24T19:11:51.255ZDownstairs last five completed:
148702024-01-24T19:11:51.255ZUpstairs last five completed: 30 29 28 27 26
148712024-01-24T19:11:51.256ZWrote out file "/tmp/test_repair_verify.data"
148722024-01-24T19:11:51.256Z
148732024-01-24T19:11:51.256ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148742024-01-24T19:11:51.256ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148752024-01-24T19:11:51.257Z 0 000-019 136 136 136 239 239 239 F T T <---
148762024-01-24T19:11:51.257Z 1 020-039 136 136 136 239 239 239 F T T <---
148772024-01-24T19:11:51.257Z 2 040-059 135 135 135 236 236 236 F T T <---
148782024-01-24T19:11:51.257Z 3 060-079 136 136 136 237 237 237 F F F
148792024-01-24T19:11:51.257Z 4 080-099 133 133 133 229 229 229 F F F
148802024-01-24T19:11:51.257Z 5 100-119 136 136 136 237 237 237 F F F
148812024-01-24T19:11:51.257Z 6 120-139 136 136 136 239 239 239 F F F
148822024-01-24T19:11:51.257Z 7 140-159 136 136 136 240 240 240 F F F
148832024-01-24T19:11:51.257Z 8 160-179 136 138 138 240 242 242 T T T <---
148842024-01-24T19:11:51.257Z 9 180-199 133 133 133 229 229 229 F F F
148852024-01-24T19:11:51.257Z 10 200-219 129 129 129 222 222 222 F F F
148862024-01-24T19:11:51.257Z 11 220-239 135 135 135 236 236 236 F F F
148872024-01-24T19:11:51.257Z 12 240-259 130 130 130 225 225 225 F F F
148882024-01-24T19:11:51.257Z 13 260-279 137 138 138 241 242 242 T T T <---
148892024-01-24T19:11:51.257Z 14 280-299 129 138 138 222 243 243 T F F <---
148902024-01-24T19:11:51.257Z 15 300-319 136 136 136 240 240 240 F F F
148912024-01-24T19:11:51.257Z 16 320-339 124 138 138 212 243 243 F F F <---
148922024-01-24T19:11:51.257Z 17 340-359 136 136 136 240 240 240 F F F
148932024-01-24T19:11:51.257Z 18 360-379 136 136 136 237 237 237 F F F
148942024-01-24T19:11:51.257Z 19 380-399 136 138 138 240 243 243 F F F <---
148952024-01-24T19:11:51.257Z 20 400-419 136 136 136 240 240 240 F F F
148962024-01-24T19:11:51.257Z 21 420-439 137 137 137 241 241 241 F F F
148972024-01-24T19:11:51.258Z 22 440-459 137 137 137 241 241 241 F F F
148982024-01-24T19:11:51.258Z 23 460-479 129 129 129 222 222 222 F F F
148992024-01-24T19:11:51.258Z 24 480-499 136 136 136 240 240 240 F T T <---
149002024-01-24T19:11:51.258Z 25 500-519 126 126 126 217 217 217 F F F
149012024-01-24T19:11:51.258Z 26 520-539 129 129 129 222 222 222 F F F
149022024-01-24T19:11:51.258Z 27 540-559 134 134 134 233 233 233 F F F
149032024-01-24T19:11:51.258Z 28 560-579 133 133 133 229 229 229 F F F
149042024-01-24T19:11:51.258Z 29 580-599 131 131 131 227 227 227 F T T <---
149052024-01-24T19:11:51.258ZMax gen: 138, Max flush: 243
149062024-01-24T19:11:51.258ZA Difference in extent metadata is expected here
149072024-01-24T19:11:51.258ZOn loop 69
149082024-01-24T19:11:51.258Z
149092024-01-24T19:11:51.258ZVerifying data now
149102024-01-24T19:11:55.440ZNo change after verify
149112024-01-24T19:11:55.440ZLoop: 69 Downstairs dump after verify (and repair):
149122024-01-24T19:11:55.464ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149132024-01-24T19:11:55.464Z 0 000-019 139 139 139 244 244 244 F F F
149142024-01-24T19:11:55.464Z 1 020-039 139 139 139 244 244 244 F F F
149152024-01-24T19:11:55.464Z 2 040-059 139 139 139 244 244 244 F F F
149162024-01-24T19:11:55.464Z 3 060-079 136 136 136 237 237 237 F F F
149172024-01-24T19:11:55.464Z 4 080-099 133 133 133 229 229 229 F F F
149182024-01-24T19:11:55.464Z 5 100-119 136 136 136 237 237 237 F F F
149192024-01-24T19:11:55.464Z 6 120-139 136 136 136 239 239 239 F F F
149202024-01-24T19:11:55.464Z 7 140-159 136 136 136 240 240 240 F F F
149212024-01-24T19:11:55.464Z 8 160-179 139 139 139 244 244 244 F F F
149222024-01-24T19:11:55.464Z 9 180-199 133 133 133 229 229 229 F F F
149232024-01-24T19:11:55.464Z 10 200-219 129 129 129 222 222 222 F F F
149242024-01-24T19:11:55.464Z 11 220-239 135 135 135 236 236 236 F F F
149252024-01-24T19:11:55.468Z 12 240-259 130 130 130 225 225 225 F F F
149262024-01-24T19:11:55.468Z 13 260-279 139 139 139 244 244 244 F F F
149272024-01-24T19:11:55.468Z 14 280-299 138 138 138 243 243 243 F F F
149282024-01-24T19:11:55.468Z 15 300-319 136 136 136 240 240 240 F F F
149292024-01-24T19:11:55.468Z 16 320-339 138 138 138 243 243 243 F F F
149302024-01-24T19:11:55.468Z 17 340-359 136 136 136 240 240 240 F F F
149312024-01-24T19:11:55.469Z 18 360-379 136 136 136 237 237 237 F F F
149322024-01-24T19:11:55.469Z 19 380-399 138 138 138 243 243 243 F F F
149332024-01-24T19:11:55.469Z 20 400-419 136 136 136 240 240 240 F F F
149342024-01-24T19:11:55.469Z 21 420-439 137 137 137 241 241 241 F F F
149352024-01-24T19:11:55.469Z 22 440-459 137 137 137 241 241 241 F F F
149362024-01-24T19:11:55.469Z 23 460-479 129 129 129 222 222 222 F F F
149372024-01-24T19:11:55.469Z 24 480-499 139 139 139 244 244 244 F F F
149382024-01-24T19:11:55.469Z 25 500-519 126 126 126 217 217 217 F F F
149392024-01-24T19:11:55.469Z 26 520-539 129 129 129 222 222 222 F F F
149402024-01-24T19:11:55.469Z 27 540-559 134 134 134 233 233 233 F F F
149412024-01-24T19:11:55.469Z 28 560-579 133 133 133 229 229 229 F F F
149422024-01-24T19:11:55.469Z 29 580-599 139 139 139 244 244 244 F F F
149432024-01-24T19:11:55.469ZMax gen: 139, Max flush: 244
149442024-01-24T19:11:55.473Z
149452024-01-24T19:11:55.473ZBegin loop 70
149462024-01-24T19:11:55.473ZDownstairs to restart: 2
149472024-01-24T19:11:55.502Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172251819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689}
149482024-01-24T19:11:55.503Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172734709Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689}
149492024-01-24T19:11:55.503Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172774993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689}
149502024-01-24T19:11:55.503Z{"msg":"Crucible 75f0ddda-0d6d-4f73-b560-7f0607beee5a has session id: 3ab308fc-69e4-4a03-aaae-de8bd927df0b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172851432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149512024-01-24T19:11:55.510Z{"msg":"Upstairs opts: Upstairs UUID: 75f0ddda-0d6d-4f73-b560-7f0607beee5a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172907569Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149522024-01-24T19:11:55.510Z{"msg":"Crucible stats registered with UUID: 75f0ddda-0d6d-4f73-b560-7f0607beee5a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.172977333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149532024-01-24T19:11:55.510ZCrucible runtime is spawned
149542024-01-24T19:11:55.510Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173311838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689}
149552024-01-24T19:11:55.510Z{"msg":"75f0ddda-0d6d-4f73-b560-7f0607beee5a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173568942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149562024-01-24T19:11:55.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173633132Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149572024-01-24T19:11:55.510Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173723611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149582024-01-24T19:11:55.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173757953Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149592024-01-24T19:11:55.510Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173804094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149602024-01-24T19:11:55.510Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173836233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149612024-01-24T19:11:55.511Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173859712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149622024-01-24T19:11:55.512Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.173956335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149632024-01-24T19:11:55.512Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.174026203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149642024-01-24T19:11:55.512Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.174075006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149652024-01-24T19:11:55.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.174120973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149662024-01-24T19:11:55.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.174518359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149672024-01-24T19:11:55.512Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.174600127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149682024-01-24T19:11:55.512Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.175362224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149692024-01-24T19:11:55.512Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.175491271Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149702024-01-24T19:11:55.512Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.17559295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149712024-01-24T19:11:55.512Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.175676209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149722024-01-24T19:11:55.512Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.176287825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149732024-01-24T19:11:55.512Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.176495361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149742024-01-24T19:11:55.512Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.176651766Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149752024-01-24T19:11:55.512Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.176799563Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149762024-01-24T19:11:55.512Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177270085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149772024-01-24T19:11:55.512Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.17735417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149782024-01-24T19:11:55.512Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177387645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149792024-01-24T19:11:55.512Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177570585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149802024-01-24T19:11:55.512Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177663605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149812024-01-24T19:11:55.512Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177728501Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149822024-01-24T19:11:55.513Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.177759725Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149832024-01-24T19:11:55.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178499805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149842024-01-24T19:11:55.513Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178585513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149852024-01-24T19:11:55.513Z{"msg":"[0]R flush_numbers[0..12]: [244, 244, 244, 237, 229, 237, 239, 240, 244, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178690256Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149862024-01-24T19:11:55.513Z{"msg":"[0]R generation[0..12]: [139, 139, 139, 136, 133, 136, 136, 136, 139, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178725272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149872024-01-24T19:11:55.513Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178749873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149882024-01-24T19:11:55.513Z{"msg":"[1]R flush_numbers[0..12]: [244, 244, 244, 237, 229, 237, 239, 240, 244, 229, 222, 236]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
149892024-01-24T19:11:55.513Z,"time":"2024-01-24T19:11:54.178796155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149902024-01-24T19:11:55.513Z{"msg":"[1]R generation[0..12]: [139, 139, 139, 136, 133, 136, 136, 136, 139, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178824343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149912024-01-24T19:11:55.513Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178869634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149922024-01-24T19:11:55.513Z{"msg":"[2]R flush_numbers[0..12]: [244, 244, 244, 237, 229, 237, 239, 240, 244, 229, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178896859Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149932024-01-24T19:11:55.513Z{"msg":"[2]R generation[0..12]: [139, 139, 139, 136, 133, 136, 136, 136, 139, 133, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178941167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149942024-01-24T19:11:55.513Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.178968285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149952024-01-24T19:11:55.513Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179011707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149962024-01-24T19:11:55.513Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179037708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149972024-01-24T19:11:55.513Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179059273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149982024-01-24T19:11:55.513Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179112796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
149992024-01-24T19:11:55.515Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179138397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150002024-01-24T19:11:55.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179180237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150012024-01-24T19:11:55.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179205552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150022024-01-24T19:11:55.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179262602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150032024-01-24T19:11:55.516Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179299465Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150042024-01-24T19:11:55.520Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179321203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150052024-01-24T19:11:55.520Z{"msg":"75f0ddda-0d6d-4f73-b560-7f0607beee5a is now active with session: 3ab308fc-69e4-4a03-aaae-de8bd927df0b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179366959Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150062024-01-24T19:11:55.520Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179420606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689,"session_id":"3ab308fc-69e4-4a03-aaae-de8bd927df0b"}
150072024-01-24T19:11:55.520Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:54.179446681Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1689}
150082024-01-24T19:11:55.520ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150092024-01-24T19:11:55.520ZLoading write count information from file "/tmp/test_repair_verify.data"
150102024-01-24T19:11:55.520ZRun Repair workload
150112024-01-24T19:11:55.520Z01/30 Write block 411 len 36864 data: 10 9 9 6 6 5 7 9 11
150122024-01-24T19:11:55.520Z02/30 Read block 289 len 20480
150132024-01-24T19:11:55.520Z03/30 Read block 576 len 36864
150142024-01-24T19:11:55.520Z04/30 Read block 228 len 20480
150152024-01-24T19:11:55.520Z05/30 Read block 136 len 16384
150162024-01-24T19:11:55.521Z06/30 Write block 563 len 12288 data: 9 8 11
150172024-01-24T19:11:55.521Z07/30 Write block 528 len 12288 data: 15 13 13
150182024-01-24T19:11:55.521Z08/30 Write block 189 len 16384 data: 9 9 9 10
150192024-01-24T19:11:55.524Z09/30 Read block 16 len 36864
150202024-01-24T19:11:55.524Z10/30 Read block 592 len 28672
150212024-01-24T19:11:55.527Z11/30 Read block 136 len 4096
150222024-01-24T19:11:55.528Z12/30 Read block 537 len 32768
150232024-01-24T19:11:55.528Z13/30 Read block 22 len 40960
150242024-01-24T19:11:55.531Z14/30 Read block 47 len 24576
150252024-01-24T19:11:55.534Z15/30 Read block 588 len 20480
150262024-01-24T19:11:55.534Z16/30 Write block 298 len 16384 data: 9 8 8 8
150272024-01-24T19:11:55.545Z17/30 Write block 248 len 8192 data: 15 17
150282024-01-24T19:11:55.545Z18/30 Write block 451 len 4096 data: 7
150292024-01-24T19:11:55.546Z19/30 Read block 86 len 36864
150302024-01-24T19:11:55.546Z20/30 Read block 115 len 4096
150312024-01-24T19:11:55.546Z21/30 Read block 337 len 8192
150322024-01-24T19:11:55.546Z22/30 Flush
150332024-01-24T19:11:55.561Z23/30 Write block 254 len 16384 data: 8 10 10 8
150342024-01-24T19:11:55.561Z24/30 Read block 26 len 40960
150352024-01-24T19:11:55.561Z25/30 Write block 77 len 8192 data: 9 10
150362024-01-24T19:11:55.562Z26/30 Read block 450 len 8192
150372024-01-24T19:11:55.562Z27/30 Read block 392 len 28672
150382024-01-24T19:11:55.562Z28/30 Read block 14 len 32768
150392024-01-24T19:11:55.562Z29/30 Write block 368 len 16384 data: 12 12 12 11
150402024-01-24T19:11:55.563Z30/30 Write block 381 len 20480 data: 12 10 11 9 7
150412024-01-24T19:11:55.563Z----------------------------------------------------------------
150422024-01-24T19:11:55.563Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
150432024-01-24T19:11:55.563ZStates: Active Active Active
150442024-01-24T19:11:55.563ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150452024-01-24T19:11:55.564Z 1 Acked 1000 Write 9 Done Done Done false
150462024-01-24T19:11:55.564Z 2 Acked 1001 Read 5 Done Done Done false
150472024-01-24T19:11:55.564Z 3 Acked 1002 Read 9 Done Done Done false
150482024-01-24T19:11:55.564Z 4 Acked 1003 Read 5 Done Done Sent false
150492024-01-24T19:11:55.565Z 5 Acked 1004 Read 4 Done Done Sent false
150502024-01-24T19:11:55.565Z 6 Acked 1005 Write 3 Done Done Sent false
150512024-01-24T19:11:55.565Z 7 Acked 1006 Write 3 Done Done Sent false
150522024-01-24T19:11:55.565Z 8 Acked 1007 Write 4 Done Done Sent false
150532024-01-24T19:11:55.565Z 9 Acked 1008 Read 9 Done Done Sent false
150542024-01-24T19:11:55.566Z 10 Acked 1009 Read 7 Done Done Sent false
150552024-01-24T19:11:55.566Z 11 Acked 1010 Read 1 Done Done Sent false
150562024-01-24T19:11:55.566Z 12 Acked 1011 Read 8 Done Done Sent false
150572024-01-24T19:11:55.566Z 13 Acked 1012 Read 10 Done Done Sent false
150582024-01-24T19:11:55.567Z 14 Acked 1013 Read 6 Done Done Sent false
150592024-01-24T19:11:55.567Z 15 Acked 1014 Read 5 Done Done Sent false
150602024-01-24T19:11:55.567Z 16 Acked 1015 Write 4 Done Done Sent false
150612024-01-24T19:11:55.567Z 17 Acked 1016 Write 2 Done Done Sent false
150622024-01-24T19:11:55.567Z 18 Acked 1017 Write 1 Done Done Sent false
150632024-01-24T19:11:55.567Z 19 Acked 1018 Read 9 Done Done Sent false
150642024-01-24T19:11:55.568Z 20 Acked 1019 Read 1 Done Done Sent false
150652024-01-24T19:11:55.568Z 21 Acked 1020 Read 2 Done Done Sent false
150662024-01-24T19:11:55.568Z 22 Acked 1021 Flush 0 Done Done Sent false
150672024-01-24T19:11:55.568Z 23 Acked 1022 Write 4 Done Done Sent false
150682024-01-24T19:11:55.568Z 24 Acked 1023 Read 10 Done Done Sent false
150692024-01-24T19:11:55.568Z 25 Acked 1024 Write 2 Done Done Sent false
150702024-01-24T19:11:55.568Z 26 Acked 1025 Read 2 Done Done Sent false
150712024-01-24T19:11:55.569Z 27 Acked 1026 Read 7 Done Done Sent false
150722024-01-24T19:11:55.569Z 28 Acked 1027 Read 8 Done Done Sent false
150732024-01-24T19:11:55.569Z 29 Acked 1028 Write 4 Done Done Sent false
150742024-01-24T19:11:55.569Z 30 Acked 1029 Write 5 Sent Sent Sent false
150752024-01-24T19:11:55.569Z STATES DS:0 DS:1 DS:2 TOTAL
150762024-01-24T19:11:55.569Z New 0 0 0 0
150772024-01-24T19:11:55.569Z Sent 1 1 27 29
150782024-01-24T19:11:55.569Z Done 29 29 3 61
150792024-01-24T19:11:55.570Z Skipped 0 0 0 0
150802024-01-24T19:11:55.570Z Error 0 0 0 0
150812024-01-24T19:11:55.570ZLast Flush: 1021 1021 0
150822024-01-24T19:11:55.583ZDownstairs last five completed:
150832024-01-24T19:11:55.583ZUpstairs last five completed: 30 29 28 27 26
150842024-01-24T19:11:55.583ZWrote out file "/tmp/test_repair_verify.data"
150852024-01-24T19:11:55.584Z
150862024-01-24T19:11:55.584ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150872024-01-24T19:11:55.601ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150882024-01-24T19:11:55.601Z 0 000-019 139 139 139 244 244 244 F F F
150892024-01-24T19:11:55.602Z 1 020-039 139 139 139 244 244 244 F F F
150902024-01-24T19:11:55.602Z 2 040-059 139 139 139 244 244 244 F F F
150912024-01-24T19:11:55.602Z 3 060-079 136 136 136 237 237 237 T T F <---
150922024-01-24T19:11:55.603Z 4 080-099 133 133 133 229 229 229 F F F
150932024-01-24T19:11:56.023Z 5 100-119 136 136 136 237 237 237 F F F
150942024-01-24T19:11:56.023Z 6 120-139 136 136 136 239 239 239 F F F
150952024-01-24T19:11:56.024Z 7 140-159 136 136 136 240 240 240 F F F
150962024-01-24T19:11:56.024Z 8 160-179 139 139 139 244 244 244 F F F
150972024-01-24T19:11:56.025Z 9 180-199 140 140 133 245 245 229 F F F <---
150982024-01-24T19:11:56.025Z 10 200-219 129 129 129 222 222 222 F F F
150992024-01-24T19:11:56.025Z 11 220-239 135 135 135 236 236 236 F F F
151002024-01-24T19:11:56.025Z 12 240-259 140 140 130 245 245 225 T T F <---
151012024-01-24T19:11:56.025Z 13 260-279 139 139 139 244 244 244 F F F
151022024-01-24T19:11:56.025Z 14 280-299 140 140 138 245 245 243 F F F <---
151032024-01-24T19:11:56.025Z 15 300-319 140 140 136 245 245 240 F F F <---
151042024-01-24T19:11:56.025Z 16 320-339 138 138 138 243 243 243 F F F
151052024-01-24T19:11:56.025Z 17 340-359 136 136 136 240 240 240 F F F
151062024-01-24T19:11:56.025Z 18 360-379 136 136 136 237 237 237 T T F <---
151072024-01-24T19:11:56.025Z 19 380-399 138 138 138 243 243 243 T T F <---
151082024-01-24T19:11:56.025Z 20 400-419 140 140 136 245 245 240 F F T <---
151092024-01-24T19:11:56.025Z 21 420-439 137 137 137 241 241 241 F F F
151102024-01-24T19:11:56.025Z 22 440-459 140 140 137 245 245 241 F F F <---
151112024-01-24T19:11:56.026Z 23 460-479 129 129 129 222 222 222 F F F
151122024-01-24T19:11:56.026Z 24 480-499 139 139 139 244 244 244 F F F
151132024-01-24T19:11:56.026Z 25 500-519 126 126 126 217 217 217 F F F
151142024-01-24T19:11:56.026Z 26 520-539 140 140 129 245 245 222 F F F <---
151152024-01-24T19:11:56.026Z 27 540-559 134 134 134 233 233 233 F F F
151162024-01-24T19:11:56.026Z 28 560-579 140 140 133 245 245 229 F F F <---
151172024-01-24T19:11:56.026Z 29 580-599 139 139 139 244 244 244 F F F
151182024-01-24T19:11:56.026ZMax gen: 140, Max flush: 245
151192024-01-24T19:11:56.026ZError: Difference in extent metadata found!
151202024-01-24T19:11:56.026ZA Difference in extent metadata is expected here
151212024-01-24T19:11:56.026ZOn loop 70
151222024-01-24T19:11:56.026Z
151232024-01-24T19:11:56.026ZVerifying data now
151242024-01-24T19:12:00.608ZNo change after verify
151252024-01-24T19:12:00.608ZLoop: 70 Downstairs dump after verify (and repair):
151262024-01-24T19:12:00.632ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151272024-01-24T19:12:00.632Z 0 000-019 139 139 139 244 244 244 F F F
151282024-01-24T19:12:00.632Z 1 020-039 139 139 139 244 244 244 F F F
151292024-01-24T19:12:00.632Z 2 040-059 139 139 139 244 244 244 F F F
151302024-01-24T19:12:00.632Z 3 060-079 141 141 141 246 246 246 F F F
151312024-01-24T19:12:00.632Z 4 080-099 133 133 133 229 229 229 F F F
151322024-01-24T19:12:00.632Z 5 100-119 136 136 136 237 237 237 F F F
151332024-01-24T19:12:00.632Z 6 120-139 136 136 136 239 239 239 F F F
151342024-01-24T19:12:00.632Z 7 140-159 136 136 136 240 240 240 F F F
151352024-01-24T19:12:00.635Z 8 160-179 139 139 139 244 244 244 F F F
151362024-01-24T19:12:00.635Z 9 180-199 140 140 140 245 245 245 F F F
151372024-01-24T19:12:00.635Z 10 200-219 129 129 129 222 222 222 F F F
151382024-01-24T19:12:00.636Z 11 220-239 135 135 135 236 236 236 F F F
151392024-01-24T19:12:00.636Z 12 240-259 141 141 141 246 246 246 F F F
151402024-01-24T19:12:00.636Z 13 260-279 139 139 139 244 244 244 F F F
151412024-01-24T19:12:00.636Z 14 280-299 140 140 140 245 245 245 F F F
151422024-01-24T19:12:00.636Z 15 300-319 140 140 140 245 245 245 F F F
151432024-01-24T19:12:00.636Z 16 320-339 138 138 138 243 243 243 F F F
151442024-01-24T19:12:00.636Z 17 340-359 136 136 136 240 240 240 F F F
151452024-01-24T19:12:00.636Z 18 360-379 141 141 141 246 246 246 F F F
151462024-01-24T19:12:00.636Z 19 380-399 141 141 141 246 246 246 F F F
151472024-01-24T19:12:00.637Z 20 400-419 140 140 140 245 245 245 F F F
151482024-01-24T19:12:00.637Z 21 420-439 137 137 137 241 241 241 F F F
151492024-01-24T19:12:00.637Z 22 440-459 140 140 140 245 245 245 F F F
151502024-01-24T19:12:00.637Z 23 460-479 129 129 129 222 222 222 F F F
151512024-01-24T19:12:00.637Z 24 480-499 139 139 139 244 244 244 F F F
151522024-01-24T19:12:00.637Z 25 500-519 126 126 126 217 217 217 F F F
151532024-01-24T19:12:00.637Z 26 520-539 140 140 140 245 245 245 F F F
151542024-01-24T19:12:00.637Z 27 540-559 134 134 134 233 233 233 F F F
151552024-01-24T19:12:00.637Z 28 560-579 140 140 140 245 245 245 F F F
151562024-01-24T19:12:00.637Z 29 580-599 139 139 139 244 244 244 F F F
151572024-01-24T19:12:00.637ZMax gen: 141, Max flush: 246
151582024-01-24T19:12:00.643Z
151592024-01-24T19:12:00.643ZBegin loop 71
151602024-01-24T19:12:00.643ZDownstairs to restart: 1
151612024-01-24T19:12:00.670Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.343184399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697}
151622024-01-24T19:12:00.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.343680446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697}
151632024-01-24T19:12:00.676Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.343768852Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697}
151642024-01-24T19:12:00.677Z{"msg":"Crucible 9f1b042a-0623-4c8e-bd9c-a93e4d412feb has session id: 061239ab-8381-4668-86b3-7015dda7694a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.34380951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151652024-01-24T19:12:00.677Z{"msg":"Upstairs opts: Upstairs UUID: 9f1b042a-0623-4c8e-bd9c-a93e4d412feb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.343902395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151662024-01-24T19:12:00.677Z{"msg":"Crucible stats registered with UUID: 9f1b042a-0623-4c8e-bd9c-a93e4d412feb","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.343948951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151672024-01-24T19:12:00.677ZCrucible runtime is spawned
151682024-01-24T19:12:00.677Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344353074Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697}
151692024-01-24T19:12:00.677Z{"msg":"9f1b042a-0623-4c8e-bd9c-a93e4d412feb active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344448657Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151702024-01-24T19:12:00.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344571848Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151712024-01-24T19:12:00.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344718827Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151722024-01-24T19:12:00.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344822909Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151732024-01-24T19:12:00.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344949003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151742024-01-24T19:12:00.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.344995808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151752024-01-24T19:12:00.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345037645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151762024-01-24T19:12:00.678Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345078791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151772024-01-24T19:12:00.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.34511887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151782024-01-24T19:12:00.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345170513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151792024-01-24T19:12:00.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345245323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151802024-01-24T19:12:00.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345287123Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151812024-01-24T19:12:00.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.345448814Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151822024-01-24T19:12:00.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.346122057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151832024-01-24T19:12:00.679Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.34625601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151842024-01-24T19:12:00.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.346325954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151852024-01-24T19:12:00.679Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.346370265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151862024-01-24T19:12:00.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.347296545Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151872024-01-24T19:12:00.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.347364928Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151882024-01-24T19:12:00.679Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.347394985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151892024-01-24T19:12:00.685Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.347565865Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151902024-01-24T19:12:00.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348596469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151912024-01-24T19:12:00.685Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348638253Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151922024-01-24T19:12:00.685Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348662114Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151932024-01-24T19:12:00.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348682694Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151942024-01-24T19:12:00.685Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348729219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151952024-01-24T19:12:00.685Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348755326Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151962024-01-24T19:12:00.685Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.348777521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151972024-01-24T19:12:00.685Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349690149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151982024-01-24T19:12:00.685Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349755654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
151992024-01-24T19:12:00.685Z{"msg":"[0]R flush_numbers[0..12]: [244, 244, 244, 246, 229, 237, 239, 240, 244, 245, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349785575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152002024-01-24T19:12:00.685Z{"msg":"[0]R generation[0..12]: [139, 139, 139, 141, 133, 136, 136, 136, 139, 140, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349831217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152012024-01-24T19:12:00.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349859354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152022024-01-24T19:12:00.685Z{"msg":"[1]R flush_numbers[0..12]: [244, 244, 244, 246, 229, 237, 239, 240, 244, 245, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349905036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152032024-01-24T19:12:00.686Z{"msg":"[1]R generation[0..12]: [139, 139, 139, 141, 133, 136, 136, 136, 139, 140, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.349932333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152042024-01-24T19:12:00.686Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
152052024-01-24T19:12:00.686Z,"time":"2024-01-24T19:11:59.349976885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152062024-01-24T19:12:00.686Z{"msg":"[2]R flush_numbers[0..12]: [244, 244, 244, 246, 229, 237, 239, 240, 244, 245, 222, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350005176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152072024-01-24T19:12:00.686Z{"msg":"[2]R generation[0..12]: [139, 139, 139, 141, 133, 136, 136, 136, 139, 140, 129, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350050939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152082024-01-24T19:12:00.686Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350078521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152092024-01-24T19:12:00.686Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350121539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152102024-01-24T19:12:00.686Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350147687Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152112024-01-24T19:12:00.686Z{"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350189795Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152122024-01-24T19:12:00.686Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350214993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152132024-01-24T19:12:00.687Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350237467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152142024-01-24T19:12:00.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350278475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152152024-01-24T19:12:00.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350304168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152162024-01-24T19:12:00.688Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350347207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152172024-01-24T19:12:00.688Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350373133Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152182024-01-24T19:12:00.688Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350415924Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152192024-01-24T19:12:00.688Z{"msg":"9f1b042a-0623-4c8e-bd9c-a93e4d412feb is now active with session: 061239ab-8381-4668-86b3-7015dda7694a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350441944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152202024-01-24T19:12:00.688Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152212024-01-24T19:12:00.688Z,"time":"2024-01-24T19:11:59.350485991Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697,"session_id":"061239ab-8381-4668-86b3-7015dda7694a"}
152222024-01-24T19:12:00.688Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:11:59.350511038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1697}
152232024-01-24T19:12:00.688ZLoading write count information from file "/tmp/test_repair_verify.data"
152242024-01-24T19:12:00.688ZRun Repair workload
152252024-01-24T19:12:00.689Z01/30 Write block 196 len 4096 data: 11
152262024-01-24T19:12:00.689Z02/30 Write block 310 len 40960 data: 8 8 8 11 13 14 15 14 12 13
152272024-01-24T19:12:00.689Z03/30 Read block 502 len 16384
152282024-01-24T19:12:00.692Z04/30 Write block 165 len 40960 data: 14 12 15 16 14 13 13 14 12 15
152292024-01-24T19:12:00.692Z05/30 Write block 66 len 8192 data: 7 7
152302024-01-24T19:12:00.693Z06/30 Read block 325 len 16384
152312024-01-24T19:12:00.693Z07/30 Read block 129 len 36864
152322024-01-24T19:12:00.693Z08/30 Read block 335 len 12288
152332024-01-24T19:12:00.693Z09/30 Read block 104 len 12288
152342024-01-24T19:12:00.696Z10/30 Read block 550 len 24576
152352024-01-24T19:12:00.696Z11/30 Read block 125 len 8192
152362024-01-24T19:12:00.699Z12/30 Read block 554 len 36864
152372024-01-24T19:12:00.699Z13/30 Flush
152382024-01-24T19:12:00.702Z14/30 Write block 134 len 4096 data: 12
152392024-01-24T19:12:00.705Z15/30 Write block 484 len 20480 data: 6 6 7 5 7
152402024-01-24T19:12:00.705Z16/30 Read block 527 len 20480
152412024-01-24T19:12:00.705Z17/30 Read block 161 len 12288
152422024-01-24T19:12:00.708Z18/30 Read block 328 len 4096
152432024-01-24T19:12:00.708Z19/30 Read block 164 len 4096
152442024-01-24T19:12:00.708Z20/30 Read block 83 len 4096
152452024-01-24T19:12:00.711Z21/30 Write block 209 len 20480 data: 9 11 9 9 8
152462024-01-24T19:12:00.711Z22/30 Write block 406 len 12288 data: 10 10 12
152472024-01-24T19:12:00.714Z23/30 Write block 159 len 32768 data: 10 9 17 16 17 13 15 13
152482024-01-24T19:12:00.714Z24/30 Write block 423 len 32768 data: 8 7 7 6 10 11 14 15
152492024-01-24T19:12:00.714Z25/30 Read block 541 len 4096
152502024-01-24T19:12:00.717Z26/30 Read block 74 len 36864
152512024-01-24T19:12:00.721Z27/30 Read block 80 len 36864
152522024-01-24T19:12:00.721Z28/30 Read block 443 len 32768
152532024-01-24T19:12:00.722Z29/30 Read block 226 len 4096
152542024-01-24T19:12:00.725Z30/30 Write block 166 len 40960 data: 14 16 17 15 14 14 15 13 16 13
152552024-01-24T19:12:00.725Z----------------------------------------------------------------
152562024-01-24T19:12:00.725Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
152572024-01-24T19:12:00.725ZStates: Active Active Active
152582024-01-24T19:12:00.725ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152592024-01-24T19:12:00.725Z 1 Acked 1000 Write 1 Done Done Done false
152602024-01-24T19:12:00.725Z 2 Acked 1001 Write 10 Done Done Done false
152612024-01-24T19:12:00.725Z 3 Acked 1002 Read 4 Done Done Done false
152622024-01-24T19:12:00.726Z 4 Acked 1003 Write 10 Done Done Done false
152632024-01-24T19:12:00.726Z 5 Acked 1004 Write 2 Done Done Done false
152642024-01-24T19:12:00.726Z 6 Acked 1005 Read 4 Done Done Done false
152652024-01-24T19:12:00.726Z 7 Acked 1006 Read 9 Done Done Done false
152662024-01-24T19:12:00.726Z 8 Acked 1007 Read 3 Done Sent Done false
152672024-01-24T19:12:00.726Z 9 Acked 1008 Read 3 Done Sent Done false
152682024-01-24T19:12:00.726Z 10 Acked 1009 Read 6 Done Sent Done false
152692024-01-24T19:12:00.726Z 11 Acked 1010 Read 2 Done Sent Done false
152702024-01-24T19:12:00.726Z 12 Acked 1011 Read 9 Done Sent Done false
152712024-01-24T19:12:00.726Z 13 Acked 1012 Flush 0 Done Sent Done false
152722024-01-24T19:12:00.726Z 14 Acked 1013 Write 1 Done Sent Done false
152732024-01-24T19:12:00.726Z 15 Acked 1014 Write 5 Done Sent Done false
152742024-01-24T19:12:00.726Z 16 Acked 1015 Read 5 Done Sent Done false
152752024-01-24T19:12:00.726Z 17 Acked 1016 Read 3 Done Sent Done false
152762024-01-24T19:12:00.726Z 18 Acked 1017 Read 1 Done Sent Done false
152772024-01-24T19:12:00.726Z 19 Acked 1018 Read 1 Done Sent Done false
152782024-01-24T19:12:00.726Z 20 Acked 1019 Read 1 Done Sent Done false
152792024-01-24T19:12:00.726Z 21 Acked 1020 Write 5 Done Sent Done false
152802024-01-24T19:12:00.726Z 22 Acked 1021 Write 3 Done Sent Done false
152812024-01-24T19:12:00.726Z 23 Acked 1022 Write 8 Done Sent Done false
152822024-01-24T19:12:00.726Z 24 Acked 1023 Write 8 Done Sent Done false
152832024-01-24T19:12:00.726Z 25 Acked 1024 Read 1 Done Sent Done false
152842024-01-24T19:12:00.727Z 26 Acked 1025 Read 9 Done Sent Done false
152852024-01-24T19:12:00.727Z 27 Acked 1026 Read 9 Done Sent Done false
152862024-01-24T19:12:00.727Z 28 Acked 1027 Read 8 Done Sent Done false
152872024-01-24T19:12:00.728Z 29 Acked 1028 Read 1 Done Sent Done false
152882024-01-24T19:12:00.728Z 30 Acked 1029 Write 10 Sent Sent Sent false
152892024-01-24T19:12:00.728Z STATES DS:0 DS:1 DS:2 TOTAL
152902024-01-24T19:12:00.728Z New 0 0 0 0
152912024-01-24T19:12:00.728Z Sent 1 23 1 25
152922024-01-24T19:12:00.728Z Done 29 7 29 65
152932024-01-24T19:12:00.728Z Skipped 0 0 0 0
152942024-01-24T19:12:00.728Z Error 0 0 0 0
152952024-01-24T19:12:00.728ZLast Flush: 1012 0 1012
152962024-01-24T19:12:00.728ZDownstairs last five completed:
152972024-01-24T19:12:00.728ZUpstairs last five completed: 30 29 28 27 26
152982024-01-24T19:12:00.728ZWrote out file "/tmp/test_repair_verify.data"
152992024-01-24T19:12:00.732Z
153002024-01-24T19:12:00.738ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153012024-01-24T19:12:00.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153022024-01-24T19:12:00.767Z 0 000-019 139 139 139 244 244 244 F F F
153032024-01-24T19:12:00.767Z 1 020-039 139 139 139 244 244 244 F F F
153042024-01-24T19:12:00.767Z 2 040-059 139 139 139 244 244 244 F F F
153052024-01-24T19:12:00.805Z 3 060-079 142 141 142 247 246 247 F T F <---
153062024-01-24T19:12:00.805Z 4 080-099 133 133 133 229 229 229 F F F
153072024-01-24T19:12:00.805Z 5 100-119 136 136 136 237 237 237 F F F
153082024-01-24T19:12:00.805Z 6 120-139 136 136 136 239 239 239 T F T <---
153092024-01-24T19:12:00.805Z 7 140-159 136 136 136 240 240 240 T F T <---
153102024-01-24T19:12:00.805Z 8 160-179 142 139 142 247 244 247 T T T <---
153112024-01-24T19:12:00.805Z 9 180-199 142 140 142 247 245 247 F T F <---
153122024-01-24T19:12:00.805Z 10 200-219 129 129 129 222 222 222 T F T <---
153132024-01-24T19:12:00.805Z 11 220-239 135 135 135 236 236 236 F F F
153142024-01-24T19:12:00.805Z 12 240-259 141 141 141 246 246 246 F F F
153152024-01-24T19:12:00.805Z 13 260-279 139 139 139 244 244 244 F F F
153162024-01-24T19:12:00.805Z 14 280-299 140 140 140 245 245 245 F F F
153172024-01-24T19:12:00.805Z 15 300-319 142 140 142 247 245 247 F T F <---
153182024-01-24T19:12:00.805Z 16 320-339 138 138 138 243 243 243 F F F
153192024-01-24T19:12:00.805Z 17 340-359 136 136 136 240 240 240 F F F
153202024-01-24T19:12:00.805Z 18 360-379 141 141 141 246 246 246 F F F
153212024-01-24T19:12:00.805Z 19 380-399 141 141 141 246 246 246 F F F
153222024-01-24T19:12:00.805Z 20 400-419 140 140 140 245 245 245 T F T <---
153232024-01-24T19:12:00.809Z 21 420-439 137 137 137 241 241 241 T F T <---
153242024-01-24T19:12:00.809Z 22 440-459 140 140 140 245 245 245 F F F
153252024-01-24T19:12:00.809Z 23 460-479 129 129 129 222 222 222 F F F
153262024-01-24T19:12:00.809Z 24 480-499 139 139 139 244 244 244 T F T <---
153272024-01-24T19:12:00.809Z 25 500-519 126 126 126 217 217 217 F F F
153282024-01-24T19:12:00.809Z 26 520-539 140 140 140 245 245 245 F F F
153292024-01-24T19:12:00.809Z 27 540-559 134 134 134 233 233 233 F F F
153302024-01-24T19:12:00.809Z 28 560-579 140 140 140 245 245 245 F F F
153312024-01-24T19:12:00.809Z 29 580-599 139 139 139 244 244 244 F F F
153322024-01-24T19:12:00.810ZMax gen: 142, Max flush: 247
153332024-01-24T19:12:00.810ZError: Difference in extent metadata found!
153342024-01-24T19:12:00.814ZA Difference in extent metadata is expected here
153352024-01-24T19:12:00.815ZOn loop 71
153362024-01-24T19:12:00.815Z
153372024-01-24T19:12:00.815ZVerifying data now
153382024-01-24T19:12:06.727ZNo change after verify
153392024-01-24T19:12:06.727ZLoop: 71 Downstairs dump after verify (and repair):
153402024-01-24T19:12:06.749ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153412024-01-24T19:12:06.749Z 0 000-019 139 139 139 244 244 244 F F F
153422024-01-24T19:12:06.749Z 1 020-039 139 139 139 244 244 244 F F F
153432024-01-24T19:12:06.749Z 2 040-059 139 139 139 244 244 244 F F F
153442024-01-24T19:12:06.750Z 3 060-079 142 142 142 247 247 247 F F F
153452024-01-24T19:12:06.750Z 4 080-099 133 133 133 229 229 229 F F F
153462024-01-24T19:12:06.750Z 5 100-119 136 136 136 237 237 237 F F F
153472024-01-24T19:12:06.755Z 6 120-139 143 143 143 248 248 248 F F F
153482024-01-24T19:12:06.755Z 7 140-159 143 143 143 248 248 248 F F F
153492024-01-24T19:12:06.755Z 8 160-179 143 143 143 248 248 248 F F F
153502024-01-24T19:12:06.755Z 9 180-199 142 142 142 247 247 247 F F F
153512024-01-24T19:12:06.755Z 10 200-219 143 143 143 248 248 248 F F F
153522024-01-24T19:12:06.755Z 11 220-239 135 135 135 236 236 236 F F F
153532024-01-24T19:12:06.755Z 12 240-259 141 141 141 246 246 246 F F F
153542024-01-24T19:12:06.755Z 13 260-279 139 139 139 244 244 244 F F F
153552024-01-24T19:12:06.755Z 14 280-299 140 140 140 245 245 245 F F F
153562024-01-24T19:12:06.755Z 15 300-319 142 142 142 247 247 247 F F F
153572024-01-24T19:12:06.756Z 16 320-339 138 138 138 243 243 243 F F F
153582024-01-24T19:12:06.756Z 17 340-359 136 136 136 240 240 240 F F F
153592024-01-24T19:12:06.756Z 18 360-379 141 141 141 246 246 246 F F F
153602024-01-24T19:12:06.756Z 19 380-399 141 141 141 246 246 246 F F F
153612024-01-24T19:12:06.756Z 20 400-419 143 143 143 248 248 248 F F F
153622024-01-24T19:12:06.756Z 21 420-439 143 143 143 248 248 248 F F F
153632024-01-24T19:12:06.756Z 22 440-459 140 140 140 245 245 245 F F F
153642024-01-24T19:12:06.756Z 23 460-479 129 129 129 222 222 222 F F F
153652024-01-24T19:12:06.756Z 24 480-499 143 143 143 248 248 248 F F F
153662024-01-24T19:12:06.756Z 25 500-519 126 126 126 217 217 217 F F F
153672024-01-24T19:12:06.756Z 26 520-539 140 140 140 245 245 245 F F F
153682024-01-24T19:12:06.756Z 27 540-559 134 134 134 233 233 233 F F F
153692024-01-24T19:12:06.757Z 28 560-579 140 140 140 245 245 245 F F F
153702024-01-24T19:12:06.757Z 29 580-599 139 139 139 244 244 244 F F F
153712024-01-24T19:12:06.757ZMax gen: 143, Max flush: 248
153722024-01-24T19:12:06.757Z
153732024-01-24T19:12:06.757ZBegin loop 72
153742024-01-24T19:12:06.757ZDownstairs to restart: 1
153752024-01-24T19:12:06.783Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.455570582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705}
153762024-01-24T19:12:06.783Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456102279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705}
153772024-01-24T19:12:06.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456138178Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705}
153782024-01-24T19:12:06.788Z{"msg":"Crucible 5f9aeec0-986e-4874-b295-4473c79b5d4e has session id: 2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456258611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153792024-01-24T19:12:06.789Z{"msg":"Upstairs opts: Upstairs UUID: 5f9aeec0-986e-4874-b295-4473c79b5d4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
153802024-01-24T19:12:06.789Z,"time":"2024-01-24T19:12:05.456319568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153812024-01-24T19:12:06.789Z{"msg":"Crucible stats registered with UUID: 5f9aeec0-986e-4874-b295-4473c79b5d4e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.45644749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153822024-01-24T19:12:06.789Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456516311Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705}
153832024-01-24T19:12:06.791Z{"msg":"5f9aeec0-986e-4874-b295-4473c79b5d4e active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456678627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153842024-01-24T19:12:06.791Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456780947Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153852024-01-24T19:12:06.791Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456907219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153862024-01-24T19:12:06.791Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.456967627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153872024-01-24T19:12:06.791Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457013355Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153882024-01-24T19:12:06.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457142788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153892024-01-24T19:12:06.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457386375Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153902024-01-24T19:12:06.792Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457478214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153912024-01-24T19:12:06.792Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.45752485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153922024-01-24T19:12:06.792Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457615456Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153932024-01-24T19:12:06.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457664189Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153942024-01-24T19:12:06.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457704183Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153952024-01-24T19:12:06.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.457733056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153962024-01-24T19:12:06.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.458526233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153972024-01-24T19:12:06.795Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.458590791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153982024-01-24T19:12:06.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.458621907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
153992024-01-24T19:12:06.795Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.458706559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154002024-01-24T19:12:06.796Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.458841532Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154012024-01-24T19:12:06.796Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.45888548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154022024-01-24T19:12:06.796Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.459585536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154032024-01-24T19:12:06.796Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.459886958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154042024-01-24T19:12:06.796Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460024184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154052024-01-24T19:12:06.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460557109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154062024-01-24T19:12:06.802Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460619781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154072024-01-24T19:12:06.803Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460646946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154082024-01-24T19:12:06.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460900871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154092024-01-24T19:12:06.804Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460955582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154102024-01-24T19:12:06.804Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.460983213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154112024-01-24T19:12:06.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461029712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154122024-01-24T19:12:06.804Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461056516Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154132024-01-24T19:12:06.804Z{"msg":"[0]R flush_numbers[0..12]: [244, 244, 244, 247, 229, 237, 248, 248, 248, 247, 248, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461101296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154142024-01-24T19:12:06.804Z{"msg":"[0]R generation[0..12]: [139, 139, 139, 142, 133, 136, 143, 143, 143, 142, 143, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461128368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154152024-01-24T19:12:06.804Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461174642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154162024-01-24T19:12:06.804Z{"msg":"[1]R flush_numbers[0..12]: [244, 244, 244, 247, 229, 237, 248, 248, 248, 247, 248, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461201712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154172024-01-24T19:12:06.804Z{"msg":"[1]R generation[0..12]: [139, 139, 139, 142, 133, 136, 143, 143, 143, 142, 143, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461247399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154182024-01-24T19:12:06.804Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.46127383Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154192024-01-24T19:12:06.804Z{"msg":"[2]R flush_numbers[0..12]: [244, 244, 244, 247, 229, 237, 248, 248, 248, 247, 248, 236]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461297318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154202024-01-24T19:12:06.804Z{"msg":"[2]R generation[0..12]: [139, 139, 139, 142, 133, 136, 143, 143, 143, 142, 143, 135]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461349727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154212024-01-24T19:12:06.804Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
154222024-01-24T19:12:06.804Z"time":"2024-01-24T19:12:05.461376755Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154232024-01-24T19:12:06.805Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461402373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154242024-01-24T19:12:06.805Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461451101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154252024-01-24T19:12:06.805Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461478627Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154262024-01-24T19:12:06.805Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461500175Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154272024-01-24T19:12:06.806Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461549885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154282024-01-24T19:12:06.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461573493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154292024-01-24T19:12:06.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461595229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154302024-01-24T19:12:06.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.46161661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154312024-01-24T19:12:06.806Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461659786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154322024-01-24T19:12:06.806Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461683786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154332024-01-24T19:12:06.806Z{"msg":"5f9aeec0-986e-4874-b295-4473c79b5d4e is now active with session: 2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461705607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154342024-01-24T19:12:06.806Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461746786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705,"session_id":"2b865408-2a98-4d9f-b4b6-70aeb8e5ed0d"}
154352024-01-24T19:12:06.806Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:05.461770558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1705}
154362024-01-24T19:12:06.806ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154372024-01-24T19:12:06.806ZLoading write count information from file "/tmp/test_repair_verify.data"
154382024-01-24T19:12:06.806ZRun Repair workload
154392024-01-24T19:12:06.807Z01/30 Write block 231 len 28672 data: 11 11 13 13 9 10 9
154402024-01-24T19:12:06.807Z02/30 Write block 539 len 8192 data: 11 9
154412024-01-24T19:12:06.807Z03/30 Read block 340 len 36864
154422024-01-24T19:12:06.807Z04/30 Write block 285 len 24576 data: 11 10 11 13 12 10
154432024-01-24T19:12:06.807Z05/30 Read block 460 len 36864
154442024-01-24T19:12:06.807Z06/30 Read block 7 len 24576
154452024-01-24T19:12:06.831Z07/30 Flush
154462024-01-24T19:12:06.831Z08/30 Write block 553 len 8192 data: 10 10
154472024-01-24T19:12:06.831Z09/30 Write block 483 len 16384 data: 7 7 7 8
154482024-01-24T19:12:06.832Z10/30 Read block 425 len 12288
154492024-01-24T19:12:06.832Z11/30 Read block 180 len 32768
154502024-01-24T19:12:06.832Z12/30 Read block 231 len 12288
154512024-01-24T19:12:06.832Z13/30 Read block 16 len 12288
154522024-01-24T19:12:06.832Z14/30 Write block 236 len 16384 data: 11 10 8 8
154532024-01-24T19:12:06.832Z15/30 Write block 36 len 16384 data: 8 6 6 5
154542024-01-24T19:12:06.832Z16/30 Flush
154552024-01-24T19:12:06.843Z17/30 Write block 398 len 24576 data: 9 8 9 10 9 6
154562024-01-24T19:12:06.843Z18/30 Write block 109 len 12288 data: 16 15 14
154572024-01-24T19:12:06.844Z19/30 Write block 84 len 36864 data: 14 11 12 9 7 6 7 9 10
154582024-01-24T19:12:06.844Z20/30 Flush
154592024-01-24T19:12:06.844Z21/30 Write block 124 len 12288 data: 13 11 9
154602024-01-24T19:12:06.844Z22/30 Write block 279 len 32768 data: 16 15 15 16 15 11 12 11
154612024-01-24T19:12:06.844Z23/30 Write block 397 len 24576 data: 11 10 9 10 11 10
154622024-01-24T19:12:06.844Z24/30 Read block 394 len 24576
154632024-01-24T19:12:06.844Z25/30 Read block 525 len 16384
154642024-01-24T19:12:06.844Z26/30 Read block 471 len 40960
154652024-01-24T19:12:06.845Z27/30 Read block 249 len 32768
154662024-01-24T19:12:06.845Z28/30 Write block 255 len 36864 data: 11 11 9 9 10 9 9 9 8
154672024-01-24T19:12:06.845Z29/30 Write block 269 len 36864 data: 11 12 11 13 15 13 12 10 13
154682024-01-24T19:12:06.845Z30/30 Write block 296 len 12288 data: 11 8 10
154692024-01-24T19:12:06.850Z----------------------------------------------------------------
154702024-01-24T19:12:06.850Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
154712024-01-24T19:12:06.850ZStates: Active Active Active
154722024-01-24T19:12:06.850ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
154732024-01-24T19:12:06.850Z 1 Acked 1000 Write 7 Done Done Done false
154742024-01-24T19:12:06.850Z 2 Acked 1001 Write 2 Done Done Done false
154752024-01-24T19:12:06.851Z 3 Acked 1002 Read 9 Done Sent Done false
154762024-01-24T19:12:06.851Z 4 Acked 1003 Write 6 Done Sent Done false
154772024-01-24T19:12:06.851Z 5 Acked 1004 Read 9 Done Sent Done false
154782024-01-24T19:12:06.851Z 6 Acked 1005 Read 6 Done Sent Done false
154792024-01-24T19:12:06.852Z 7 Acked 1006 Flush 0 Done Sent Done false
154802024-01-24T19:12:06.852Z 8 Acked 1007 Write 2 Done Sent Done false
154812024-01-24T19:12:06.852Z 9 Acked 1008 Write 4 Done Sent Done false
154822024-01-24T19:12:06.852Z 10 Acked 1009 Read 3 Done Sent Done false
154832024-01-24T19:12:06.852Z 11 Acked 1010 Read 8 Done Sent Done false
154842024-01-24T19:12:06.852Z 12 Acked 1011 Read 3 Done Sent Done false
154852024-01-24T19:12:06.852Z 13 Acked 1012 Read 3 Done Sent Done false
154862024-01-24T19:12:06.852Z 14 Acked 1013 Write 4 Done Sent Done false
154872024-01-24T19:12:06.852Z 15 Acked 1014 Write 4 Done Sent Done false
154882024-01-24T19:12:06.852Z 16 Acked 1015 Flush 0 Done Sent Done false
154892024-01-24T19:12:06.853Z 17 Acked 1016 Write 6 Done Sent Done false
154902024-01-24T19:12:06.853Z 18 Acked 1017 Write 3 Done Sent Done false
154912024-01-24T19:12:06.853Z 19 Acked 1018 Write 9 Done Sent Done false
154922024-01-24T19:12:06.853Z 20 Acked 1019 Flush 0 Done Sent Done false
154932024-01-24T19:12:06.853Z 21 Acked 1020 Write 3 Done Sent Done false
154942024-01-24T19:12:06.853Z 22 Acked 1021 Write 8 Done Sent Done false
154952024-01-24T19:12:06.853Z 23 Acked 1022 Write 6 Done Sent Done false
154962024-01-24T19:12:06.853Z 24 Acked 1023 Read 6 Done Sent Done false
154972024-01-24T19:12:06.853Z 25 Acked 1024 Read 4 Done Sent Done false
154982024-01-24T19:12:06.853Z 26 Acked 1025 Read 10 Done Sent Done false
154992024-01-24T19:12:06.853Z 27 Acked 1026 Read 8 Done Sent Done false
155002024-01-24T19:12:06.853Z 28 Acked 1027 Write 9 Done Sent Done false
155012024-01-24T19:12:06.853Z 29 Acked 1028 Write 9 Sent Sent Sent false
155022024-01-24T19:12:06.854Z 30 Acked 1029 Write 3 Sent Sent Sent false
155032024-01-24T19:12:06.854Z STATES DS:0 DS:1 DS:2 TOTAL
155042024-01-24T19:12:06.854Z New 0 0 0 0
155052024-01-24T19:12:06.854Z Sent 2 28 2 32
155062024-01-24T19:12:06.854Z Done 28 2 28 58
155072024-01-24T19:12:06.854Z Skipped 0 0 0 0
155082024-01-24T19:12:06.854Z Error 0 0 0 0
155092024-01-24T19:12:06.854ZLast Flush: 1019 0 1019
155102024-01-24T19:12:06.854ZDownstairs last five completed:
155112024-01-24T19:12:06.854ZUpstairs last five completed: 30 29 28 27 26
155122024-01-24T19:12:06.855ZWrote out file "/tmp/test_repair_verify.data"
155132024-01-24T19:12:06.855Z
155142024-01-24T19:12:06.860ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155152024-01-24T19:12:06.879ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155162024-01-24T19:12:06.879Z 0 000-019 139 139 139 244 244 244 F F F
155172024-01-24T19:12:06.883Z 1 020-039 144 139 144 250 244 250 F F F <---
155182024-01-24T19:12:06.883Z 2 040-059 139 139 139 244 244 244 F F F
155192024-01-24T19:12:06.883Z 3 060-079 142 142 142 247 247 247 F F F
155202024-01-24T19:12:06.883Z 4 080-099 144 133 144 251 229 251 F F F <---
155212024-01-24T19:12:06.883Z 5 100-119 144 136 144 251 237 251 F F F <---
155222024-01-24T19:12:06.884Z 6 120-139 143 143 143 248 248 248 T F T <---
155232024-01-24T19:12:06.884Z 7 140-159 143 143 143 248 248 248 F F F
155242024-01-24T19:12:06.884Z 8 160-179 143 143 143 248 248 248 F F F
155252024-01-24T19:12:06.884Z 9 180-199 142 142 142 247 247 247 F F F
155262024-01-24T19:12:06.884Z 10 200-219 143 143 143 248 248 248 F F F
155272024-01-24T19:12:06.884Z 11 220-239 144 135 144 250 236 250 F T F <---
155282024-01-24T19:12:06.884Z 12 240-259 141 141 141 246 246 246 T F T <---
155292024-01-24T19:12:06.884Z 13 260-279 139 139 139 244 244 244 T F T <---
155302024-01-24T19:12:06.884Z 14 280-299 144 140 144 249 245 249 T F T <---
155312024-01-24T19:12:06.884Z 15 300-319 142 142 142 247 247 247 F F F
155322024-01-24T19:12:06.884Z 16 320-339 138 138 138 243 243 243 F F F
155332024-01-24T19:12:06.884Z 17 340-359 136 136 136 240 240 240 F F F
155342024-01-24T19:12:06.884Z 18 360-379 141 141 141 246 246 246 F F F
155352024-01-24T19:12:06.885Z 19 380-399 144 141 144 251 246 251 T F T <---
155362024-01-24T19:12:06.885Z 20 400-419 144 143 144 251 248 251 T F T <---
155372024-01-24T19:12:06.885Z 21 420-439 143 143 143 248 248 248 F F F
155382024-01-24T19:12:06.885Z 22 440-459 140 140 140 245 245 245 F F F
155392024-01-24T19:12:06.885Z 23 460-479 129 129 129 222 222 222 F F F
155402024-01-24T19:12:06.885Z 24 480-499 144 143 144 250 248 250 F F F <---
155412024-01-24T19:12:06.885Z 25 500-519 126 126 126 217 217 217 F F F
155422024-01-24T19:12:06.885Z 26 520-539 144 140 144 249 245 249 F T F <---
155432024-01-24T19:12:06.885Z 27 540-559 144 134 144 250 233 250 F T F <---
155442024-01-24T19:12:06.885Z 28 560-579 140 140 140 245 245 245 F F F
155452024-01-24T19:12:06.885Z 29 580-599 139 139 139 244 244 244 F F F
155462024-01-24T19:12:06.885ZMax gen: 144, Max flush: 251
155472024-01-24T19:12:06.886ZError: Difference in extent metadata found!
155482024-01-24T19:12:06.889ZA Difference in extent metadata is expected here
155492024-01-24T19:12:06.889ZOn loop 72
155502024-01-24T19:12:06.889Z
155512024-01-24T19:12:06.889ZVerifying data now
155522024-01-24T19:12:13.050ZNo change after verify
155532024-01-24T19:12:13.050ZLoop: 72 Downstairs dump after verify (and repair):
155542024-01-24T19:12:13.073ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155552024-01-24T19:12:13.073Z 0 000-019 139 139 139 244 244 244 F F F
155562024-01-24T19:12:13.073Z 1 020-039 144 144 144 250 250 250 F F F
155572024-01-24T19:12:13.073Z 2 040-059 139 139 139 244 244 244 F F F
155582024-01-24T19:12:13.073Z 3 060-079 142 142 142 247 247 247 F F F
155592024-01-24T19:12:13.073Z 4 080-099 144 144 144 251 251 251 F F F
155602024-01-24T19:12:13.073Z 5 100-119 144 144 144 251 251 251 F F F
155612024-01-24T19:12:13.073Z 6 120-139 145 145 145 252 252 252 F F F
155622024-01-24T19:12:13.073Z 7 140-159 143 143 143 248 248 248 F F F
155632024-01-24T19:12:13.077Z 8 160-179 143 143 143 248 248 248 F F F
155642024-01-24T19:12:13.077Z 9 180-199 142 142 142 247 247 247 F F F
155652024-01-24T19:12:13.077Z 10 200-219 143 143 143 248 248 248 F F F
155662024-01-24T19:12:13.077Z 11 220-239 144 144 144 250 250 250 F F F
155672024-01-24T19:12:13.077Z 12 240-259 145 145 145 252 252 252 F F F
155682024-01-24T19:12:13.077Z 13 260-279 145 145 145 252 252 252 F F F
155692024-01-24T19:12:13.079Z 14 280-299 145 145 145 252 252 252 F F F
155702024-01-24T19:12:13.079Z 15 300-319 142 142 142 247 247 247 F F F
155712024-01-24T19:12:13.079Z 16 320-339 138 138 138 243 243 243 F F F
155722024-01-24T19:12:13.079Z 17 340-359 136 136 136 240 240 240 F F F
155732024-01-24T19:12:13.079Z 18 360-379 141 141 141 246 246 246 F F F
155742024-01-24T19:12:13.079Z 19 380-399 145 145 145 252 252 252 F F F
155752024-01-24T19:12:13.080Z 20 400-419 145 145 145 252 252 252 F F F
155762024-01-24T19:12:13.080Z 21 420-439 143 143 143 248 248 248 F F F
155772024-01-24T19:12:13.080Z 22 440-459 140 140 140 245 245 245 F F F
155782024-01-24T19:12:13.080Z 23 460-479 129 129 129 222 222 222 F F F
155792024-01-24T19:12:13.080Z 24 480-499 144 144 144 250 250 250 F F F
155802024-01-24T19:12:13.080Z 25 500-519 126 126 126 217 217 217 F F F
155812024-01-24T19:12:13.080Z 26 520-539 144 144 144 249 249 249 F F F
155822024-01-24T19:12:13.080Z 27 540-559 144 144 144 250 250 250 F F F
155832024-01-24T19:12:13.080Z 28 560-579 140 140 140 245 245 245 F F F
155842024-01-24T19:12:13.080Z 29 580-599 139 139 139 244 244 244 F F F
155852024-01-24T19:12:13.080ZMax gen: 145, Max flush: 252
155862024-01-24T19:12:13.080Z
155872024-01-24T19:12:13.080ZBegin loop 73
155882024-01-24T19:12:13.080ZDownstairs to restart: 2
155892024-01-24T19:12:13.107Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.780805632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713}
155902024-01-24T19:12:13.107Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.781313009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713}
155912024-01-24T19:12:13.108Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.781388128Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713}
155922024-01-24T19:12:13.113Z{"msg":"Crucible 9d9f37f4-12ce-431c-82e2-f8f0cdf19694 has session id: d7271d7e-346c-4d16-a2c8-483ccf5cc85a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.78144797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
155932024-01-24T19:12:13.113Z{"msg":"Upstairs opts: Upstairs UUID: 9d9f37f4-12ce-431c-82e2-f8f0cdf19694, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","Crucible runtime is spawned
155942024-01-24T19:12:13.113Zv":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.781814682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
155952024-01-24T19:12:13.113Z{"msg":"Crucible stats registered with UUID: 9d9f37f4-12ce-431c-82e2-f8f0cdf19694","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.781857437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
155962024-01-24T19:12:13.114Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.781927395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713}
155972024-01-24T19:12:13.114Z{"msg":"9d9f37f4-12ce-431c-82e2-f8f0cdf19694 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782142444Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
155982024-01-24T19:12:13.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782202354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
155992024-01-24T19:12:13.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782308045Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156002024-01-24T19:12:13.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782351033Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156012024-01-24T19:12:13.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782438815Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156022024-01-24T19:12:13.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782562737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156032024-01-24T19:12:13.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782641001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156042024-01-24T19:12:13.115Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782717855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156052024-01-24T19:12:13.115Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782763386Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156062024-01-24T19:12:13.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782842561Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156072024-01-24T19:12:13.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.78289003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156082024-01-24T19:12:13.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.78296175Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156092024-01-24T19:12:13.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.782994993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156102024-01-24T19:12:13.116Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.783851388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156112024-01-24T19:12:13.118Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.783936912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156122024-01-24T19:12:13.118Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.784168438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156132024-01-24T19:12:13.118Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.784279054Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156142024-01-24T19:12:13.118Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.784874537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156152024-01-24T19:12:13.118Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785109347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156162024-01-24T19:12:13.118Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785313411Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156172024-01-24T19:12:13.118Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785369892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156182024-01-24T19:12:13.124Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785777929Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156192024-01-24T19:12:13.124Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785946461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156202024-01-24T19:12:13.124Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.785995691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156212024-01-24T19:12:13.124Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.786053111Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156222024-01-24T19:12:13.124Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.78609679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156232024-01-24T19:12:13.124Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.786164148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156242024-01-24T19:12:13.124Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.786521925Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156252024-01-24T19:12:13.124Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787221682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156262024-01-24T19:12:13.124Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787283056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156272024-01-24T19:12:13.125Z{"msg":"[0]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 248, 248, 247, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787362393Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156282024-01-24T19:12:13.125Z{"msg":"[0]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 143, 143, 142, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787395731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156292024-01-24T19:12:13.125Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787455531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156302024-01-24T19:12:13.125Z{"msg":"[1]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 248, 248, 247, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787485296Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156312024-01-24T19:12:13.125ZWait for a query_work_queue command to finish before sending IO
156322024-01-24T19:12:13.125Z{"msg":"[1]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 143, 143, 142, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787646259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156332024-01-24T19:12:13.125Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787748493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156342024-01-24T19:12:13.125Z{"msg":"[2]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 248, 248, 247, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787795918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156352024-01-24T19:12:13.125Z{"msg":"[2]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 143, 143, 142, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787856069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156362024-01-24T19:12:13.125Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787889871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156372024-01-24T19:12:13.125Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.787947024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156382024-01-24T19:12:13.125Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788017139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156392024-01-24T19:12:13.125Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788053329Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156402024-01-24T19:12:13.125Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788111249Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156412024-01-24T19:12:13.127Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.7881405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156422024-01-24T19:12:13.127Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788205875Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156432024-01-24T19:12:13.127Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788238526Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156442024-01-24T19:12:13.127Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156452024-01-24T19:12:13.127Z,"time":"2024-01-24T19:12:11.788313888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156462024-01-24T19:12:13.127Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788377083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156472024-01-24T19:12:13.127Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788407088Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156482024-01-24T19:12:13.127Z{"msg":"9d9f37f4-12ce-431c-82e2-f8f0cdf19694 is now active with session: d7271d7e-346c-4d16-a2c8-483ccf5cc85a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788462169Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156492024-01-24T19:12:13.127Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788491883Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713,"session_id":"d7271d7e-346c-4d16-a2c8-483ccf5cc85a"}
156502024-01-24T19:12:13.128Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:11.788547403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1713}
156512024-01-24T19:12:13.128ZLoading write count information from file "/tmp/test_repair_verify.data"
156522024-01-24T19:12:13.128ZRun Repair workload
156532024-01-24T19:12:13.128Z01/30 Write block 170 len 12288 data: 15 15 16
156542024-01-24T19:12:13.128Z02/30 Read block 60 len 16384
156552024-01-24T19:12:13.128Z03/30 Flush
156562024-01-24T19:12:13.132Z04/30 Write block 370 len 16384 data: 13 12 10 11
156572024-01-24T19:12:13.132Z05/30 Write block 260 len 8192 data: 10 10
156582024-01-24T19:12:13.132Z06/30 Write block 588 len 36864 data: 10 8 8 10 13 12 11 8 10
156592024-01-24T19:12:13.132Z07/30 Flush
156602024-01-24T19:12:13.132Z08/30 Write block 406 len 28672 data: 11 11 13 7 10 11 10
156612024-01-24T19:12:13.132Z09/30 Read block 557 len 28672
156622024-01-24T19:12:13.136Z10/30 Write block 147 len 24576 data: 14 13 13 11 13 15
156632024-01-24T19:12:13.136Z11/30 Write block 480 len 36864 data: 10 11 11 8 8 8 9 6 8
156642024-01-24T19:12:13.136Z12/30 Read block 77 len 24576
156652024-01-24T19:12:13.140Z13/30 Read block 381 len 20480
156662024-01-24T19:12:13.140Z14/30 Read block 139 len 20480
156672024-01-24T19:12:13.143Z15/30 Write block 347 len 36864 data: 10 8 11 9 10 9 10 9 11
156682024-01-24T19:12:13.143Z16/30 Read block 244 len 36864
156692024-01-24T19:12:13.147Z17/30 Read block 182 len 28672
156702024-01-24T19:12:13.147Z18/30 Write block 582 len 24576 data: 9 10 11 10 11 11
156712024-01-24T19:12:13.151Z19/30 Write block 156 len 40960 data: 12 14 15 11 10 18 17 18 14 16
156722024-01-24T19:12:13.151Z20/30 Write block 182 len 12288 data: 8 8 10
156732024-01-24T19:12:13.151Z21/30 Read block 275 len 8192
156742024-01-24T19:12:13.155Z22/30 Flush
156752024-01-24T19:12:13.158Z23/30 Write block 256 len 12288 data: 12 10 10
156762024-01-24T19:12:13.158Z24/30 Write block 508 len 28672 data: 10 10 13 14 14 14 13
156772024-01-24T19:12:13.158Z25/30 Write block 326 len 20480 data: 8 11 13 14 12
156782024-01-24T19:12:13.162Z26/30 Read block 489 len 32768
156792024-01-24T19:12:13.162Z27/30 Write block 142 len 36864 data: 6 7 10 10 12 15 14 14 12
156802024-01-24T19:12:13.165Z28/30 Read block 580 len 24576
156812024-01-24T19:12:13.165Z29/30 Read block 439 len 8192
156822024-01-24T19:12:13.170Z30/30 Read block 108 len 4096
156832024-01-24T19:12:13.170Z----------------------------------------------------------------
156842024-01-24T19:12:13.170Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
156852024-01-24T19:12:13.170ZStates: Active Active Active
156862024-01-24T19:12:13.170ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156872024-01-24T19:12:13.170Z 1 Acked 1000 Write 3 Done Done Done false
156882024-01-24T19:12:13.170Z 2 Acked 1001 Read 4 Done Done Done false
156892024-01-24T19:12:13.170Z 3 Acked 1002 Flush 0 Done Done Sent false
156902024-01-24T19:12:13.171Z 4 Acked 1003 Write 4 Done Done Sent false
156912024-01-24T19:12:13.171Z 5 Acked 1004 Write 2 Done Done Sent false
156922024-01-24T19:12:13.171Z 6 Acked 1005 Write 9 Done Done Sent false
156932024-01-24T19:12:13.171Z 7 Acked 1006 Flush 0 Done Done Sent false
156942024-01-24T19:12:13.171Z 8 Acked 1007 Write 7 Done Done Sent false
156952024-01-24T19:12:13.171Z 9 Acked 1008 Read 7 Done Done Sent false
156962024-01-24T19:12:13.171Z 10 Acked 1009 Write 6 Done Done Sent false
156972024-01-24T19:12:13.171Z 11 Acked 1010 Write 9 Done Done Sent false
156982024-01-24T19:12:13.171Z 12 Acked 1011 Read 6 Done Done Sent false
156992024-01-24T19:12:13.171Z 13 Acked 1012 Read 5 Done Done Sent false
157002024-01-24T19:12:13.171Z 14 Acked 1013 Read 5 Done Done Sent false
157012024-01-24T19:12:13.171Z 15 Acked 1014 Write 9 Done Done Sent false
157022024-01-24T19:12:13.172Z 16 Acked 1015 Read 9 Done Done Sent false
157032024-01-24T19:12:13.172Z 17 Acked 1016 Read 7 Done Done Sent false
157042024-01-24T19:12:13.172Z 18 Acked 1017 Write 6 Done Done Sent false
157052024-01-24T19:12:13.172Z 19 Acked 1018 Write 10 Done Done Sent false
157062024-01-24T19:12:13.172Z 20 Acked 1019 Write 3 Done Done Sent false
157072024-01-24T19:12:13.173Z 21 Acked 1020 Read 2 Done Done Sent false
157082024-01-24T19:12:13.173Z 22 Acked 1021 Flush 0 Done Done Sent false
157092024-01-24T19:12:13.173Z 23 Acked 1022 Write 3 Done Done Sent false
157102024-01-24T19:12:13.173Z 24 Acked 1023 Write 7 Done Done Sent false
157112024-01-24T19:12:13.173Z 25 Acked 1024 Write 5 Done Done Sent false
157122024-01-24T19:12:13.173Z 26 Acked 1025 Read 8 Done Done Sent false
157132024-01-24T19:12:13.173Z 27 Acked 1026 Write 9 Done Done Sent false
157142024-01-24T19:12:13.173Z 28 Acked 1027 Read 6 Done Done Sent false
157152024-01-24T19:12:13.173Z 29 Acked 1028 Read 2 Done Done Sent false
157162024-01-24T19:12:13.173Z 30 Acked 1029 Read 1 Done Sent Sent false
157172024-01-24T19:12:13.173Z STATES DS:0 DS:1 DS:2 TOTAL
157182024-01-24T19:12:13.173Z New 0 0 0 0
157192024-01-24T19:12:13.173Z Sent 0 1 28 29
157202024-01-24T19:12:13.173Z Done 30 29 2 61
157212024-01-24T19:12:13.173Z Skipped 0 0 0 0
157222024-01-24T19:12:13.173Z Error 0 0 0 0
157232024-01-24T19:12:13.173ZLast Flush: 1021 1021 0
157242024-01-24T19:12:13.173ZDownstairs last five completed:
157252024-01-24T19:12:13.173ZUpstairs last five completed: 30 29 28 27 26
157262024-01-24T19:12:13.173ZWrote out file "/tmp/test_repair_verify.data"
157272024-01-24T19:12:13.177Z
157282024-01-24T19:12:13.180ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157292024-01-24T19:12:13.205ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157302024-01-24T19:12:13.205Z 0 000-019 139 139 139 244 244 244 F F F
157312024-01-24T19:12:13.208Z 1 020-039 144 144 144 250 250 250 F F F
157322024-01-24T19:12:13.208Z 2 040-059 139 139 139 244 244 244 F F F
157332024-01-24T19:12:13.209Z 3 060-079 142 142 142 247 247 247 F F F
157342024-01-24T19:12:13.209Z 4 080-099 144 144 144 251 251 251 F F F
157352024-01-24T19:12:13.209Z 5 100-119 144 144 144 251 251 251 F F F
157362024-01-24T19:12:13.209Z 6 120-139 145 145 145 252 252 252 F F F
157372024-01-24T19:12:13.209Z 7 140-159 146 146 143 255 255 248 T T F <---
157382024-01-24T19:12:13.209Z 8 160-179 146 146 143 255 255 248 F F T <---
157392024-01-24T19:12:13.209Z 9 180-199 146 146 142 255 255 247 F F F <---
157402024-01-24T19:12:13.209Z 10 200-219 143 143 143 248 248 248 F F F
157412024-01-24T19:12:13.209Z 11 220-239 144 144 144 250 250 250 F F F
157422024-01-24T19:12:13.209Z 12 240-259 145 145 145 252 252 252 T T F <---
157432024-01-24T19:12:13.209Z 13 260-279 146 146 145 254 254 252 F F F <---
157442024-01-24T19:12:13.209Z 14 280-299 145 145 145 252 252 252 F F F
157452024-01-24T19:12:13.209Z 15 300-319 142 142 142 247 247 247 F F F
157462024-01-24T19:12:13.209Z 16 320-339 138 138 138 243 243 243 T T F <---
157472024-01-24T19:12:13.209Z 17 340-359 146 146 136 255 255 240 F F F <---
157482024-01-24T19:12:13.209Z 18 360-379 146 146 141 254 254 246 F F F <---
157492024-01-24T19:12:13.209Z 19 380-399 145 145 145 252 252 252 F F F
157502024-01-24T19:12:13.211Z 20 400-419 146 146 145 255 255 252 F F F <---
157512024-01-24T19:12:13.211Z 21 420-439 143 143 143 248 248 248 F F F
157522024-01-24T19:12:13.211Z 22 440-459 140 140 140 245 245 245 F F F
157532024-01-24T19:12:13.211Z 23 460-479 129 129 129 222 222 222 F F F
157542024-01-24T19:12:13.211Z 24 480-499 146 146 144 255 255 250 F F F <---
157552024-01-24T19:12:13.211Z 25 500-519 126 126 126 217 217 217 T T F <---
157562024-01-24T19:12:13.211Z 26 520-539 144 144 144 249 249 249 F F F
157572024-01-24T19:12:13.211Z 27 540-559 144 144 144 250 250 250 F F F
157582024-01-24T19:12:13.211Z 28 560-579 140 140 140 245 245 245 F F F
157592024-01-24T19:12:13.211Z 29 580-599 146 146 139 255 255 244 F F F <---
157602024-01-24T19:12:13.211ZMax gen: 146, Max flush: 255
157612024-01-24T19:12:13.212ZError: Difference in extent metadata found!
157622024-01-24T19:12:13.215ZA Difference in extent metadata is expected here
157632024-01-24T19:12:13.215ZOn loop 73
157642024-01-24T19:12:13.215Z
157652024-01-24T19:12:13.215ZVerifying data now
157662024-01-24T19:12:18.993ZNo change after verify
157672024-01-24T19:12:18.993ZLoop: 73 Downstairs dump after verify (and repair):
157682024-01-24T19:12:19.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157692024-01-24T19:12:19.173Z 0 000-019 139 139 139 244 244 244 F F F
157702024-01-24T19:12:19.173Z 1 020-039 144 144 144 250 250 250 F F F
157712024-01-24T19:12:19.173Z 2 040-059 139 139 139 244 244 244 F F F
157722024-01-24T19:12:19.173Z 3 060-079 142 142 142 247 247 247 F F F
157732024-01-24T19:12:19.173Z 4 080-099 144 144 144 251 251 251 F F F
157742024-01-24T19:12:19.173Z 5 100-119 144 144 144 251 251 251 F F F
157752024-01-24T19:12:19.173Z 6 120-139 145 145 145 252 252 252 F F F
157762024-01-24T19:12:19.174Z 7 140-159 147 147 147 256 256 256 F F F
157772024-01-24T19:12:19.174Z 8 160-179 146 146 146 255 255 255 F F F
157782024-01-24T19:12:19.174Z 9 180-199 146 146 146 255 255 255 F F F
157792024-01-24T19:12:19.174Z 10 200-219 143 143 143 248 248 248 F F F
157802024-01-24T19:12:19.174Z 11 220-239 144 144 144 250 250 250 F F F
157812024-01-24T19:12:19.174Z 12 240-259 147 147 147 256 256 256 F F F
157822024-01-24T19:12:19.174Z 13 260-279 146 146 146 254 254 254 F F F
157832024-01-24T19:12:19.174Z 14 280-299 145 145 145 252 252 252 F F F
157842024-01-24T19:12:19.174Z 15 300-319 142 142 142 247 247 247 F F F
157852024-01-24T19:12:19.174Z 16 320-339 147 147 147 256 256 256 F F F
157862024-01-24T19:12:19.174Z 17 340-359 146 146 146 255 255 255 F F F
157872024-01-24T19:12:19.174Z 18 360-379 146 146 146 254 254 254 F F F
157882024-01-24T19:12:19.175Z 19 380-399 145 145 145 252 252 252 F F F
157892024-01-24T19:12:19.175Z 20 400-419 146 146 146 255 255 255 F F F
157902024-01-24T19:12:19.175Z 21 420-439 143 143 143 248 248 248 F F F
157912024-01-24T19:12:19.175Z 22 440-459 140 140 140 245 245 245 F F F
157922024-01-24T19:12:19.175Z 23 460-479 129 129 129 222 222 222 F F F
157932024-01-24T19:12:19.175Z 24 480-499 146 146 146 255 255 255 F F F
157942024-01-24T19:12:19.180Z 25 500-519 147 147 147 256 256 256 F F F
157952024-01-24T19:12:19.180Z 26 520-539 144 144 144 249 249 249 F F F
157962024-01-24T19:12:19.180Z 27 540-559 144 144 144 250 250 250 F F F
157972024-01-24T19:12:19.180Z 28 560-579 140 140 140 245 245 245 F F F
157982024-01-24T19:12:19.180Z 29 580-599 146 146 146 255 255 255 F F F
157992024-01-24T19:12:19.180ZMax gen: 147, Max flush: 256
158002024-01-24T19:12:19.180Z
158012024-01-24T19:12:19.180ZBegin loop 74
158022024-01-24T19:12:19.181ZDownstairs to restart: 1
158032024-01-24T19:12:19.181Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.026580661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721}
158042024-01-24T19:12:19.181Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027092144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721}
158052024-01-24T19:12:19.181Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027122351Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721}
158062024-01-24T19:12:19.181Z{"msg":"Crucible fba72456-2089-4e04-b75d-9ea081f9f7a4 has session id: 8dd592ef-6ced-4c26-8264-f6b2c6333d44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027145429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158072024-01-24T19:12:19.181Z{"msg":"Upstairs opts: Upstairs UUID: fba72456-2089-4e04-b75d-9ea081f9f7a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027226949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158082024-01-24T19:12:19.181Z{"msg":"Crucible stats registered with UUID: fba72456-2089-4e04-b75d-9ea081f9f7a4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027273566Z","Crucible runtime is spawned
158092024-01-24T19:12:19.197Zhostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158102024-01-24T19:12:19.197Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027515301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721}
158112024-01-24T19:12:19.197Z{"msg":"fba72456-2089-4e04-b75d-9ea081f9f7a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027677894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158122024-01-24T19:12:19.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.02773877Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158132024-01-24T19:12:19.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027767294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158142024-01-24T19:12:19.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027790175Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158152024-01-24T19:12:19.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027886524Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158162024-01-24T19:12:19.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027931866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158172024-01-24T19:12:19.197Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.027975699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158182024-01-24T19:12:19.197Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.028230867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158192024-01-24T19:12:19.198Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.028396032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158202024-01-24T19:12:19.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.028443051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158212024-01-24T19:12:19.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.02851442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158222024-01-24T19:12:19.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.028558712Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158232024-01-24T19:12:19.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.028663865Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158242024-01-24T19:12:19.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.029747184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158252024-01-24T19:12:19.213Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.029802407Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158262024-01-24T19:12:19.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.029846998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158272024-01-24T19:12:19.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.029953259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158282024-01-24T19:12:19.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.030876215Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158292024-01-24T19:12:19.213Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.030942846Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158302024-01-24T19:12:19.213Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.030972314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158312024-01-24T19:12:19.213Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.031151112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158322024-01-24T19:12:19.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.032073429Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158332024-01-24T19:12:19.214Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.032139517Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158342024-01-24T19:12:19.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.03226037Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158352024-01-24T19:12:19.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.032311257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158362024-01-24T19:12:19.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.03233744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158372024-01-24T19:12:19.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.032386642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158382024-01-24T19:12:19.214Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.032441602Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158392024-01-24T19:12:19.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033188049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158402024-01-24T19:12:19.248Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033321265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158412024-01-24T19:12:19.248Z{"msg":"[0]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 256, 255, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033364003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158422024-01-24T19:12:19.248Z{"msg":"[0]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 147, 146, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033436496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158432024-01-24T19:12:19.248Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033492809Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158442024-01-24T19:12:19.248Z{"msg":"[1]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 256, 255, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033520366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158452024-01-24T19:12:19.248Z{"msg":"Wait for a query_work_queue command to finish before sending IO
158462024-01-24T19:12:19.248Z[1]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 147, 146, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033567728Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158472024-01-24T19:12:19.248Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033594719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158482024-01-24T19:12:19.248Z{"msg":"[2]R flush_numbers[0..12]: [244, 250, 244, 247, 251, 251, 252, 256, 255, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033619689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158492024-01-24T19:12:19.249Z{"msg":"[2]R generation[0..12]: [139, 144, 139, 142, 144, 144, 145, 147, 146, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033642439Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158502024-01-24T19:12:19.296Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033686801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158512024-01-24T19:12:19.296Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033712197Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158522024-01-24T19:12:19.296Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033757644Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158532024-01-24T19:12:19.296Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033782328Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158542024-01-24T19:12:19.296Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033825749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158552024-01-24T19:12:19.296Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033851039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158562024-01-24T19:12:19.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033870915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158572024-01-24T19:12:19.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033912958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158582024-01-24T19:12:19.297Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033938223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158592024-01-24T19:12:19.297Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.033960803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158602024-01-24T19:12:19.297Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.034003517Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158612024-01-24T19:12:19.297Z{"msg":"fba72456-2089-4e04-b75d-9ea081f9f7a4 is now active with session: 8dd592ef-6ced-4c26-8264-f6b2c6333d44","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.034028366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158622024-01-24T19:12:19.297Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.034075599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721,"session_id":"8dd592ef-6ced-4c26-8264-f6b2c6333d44"}
158632024-01-24T19:12:19.305Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:17.034099106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1721}
158642024-01-24T19:12:19.305ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158652024-01-24T19:12:19.305ZLoading write count information from file "/tmp/test_repair_verify.data"
158662024-01-24T19:12:19.305ZRun Repair workload
158672024-01-24T19:12:19.306Z01/30 Write block 105 len 28672 data: 13 14 15 16 17 16 15
158682024-01-24T19:12:19.615Z02/30 Write block 386 len 12288 data: 7 7 8
158692024-01-24T19:12:19.615Z03/30 Read block 273 len 32768
158702024-01-24T19:12:19.615Z04/30 Read block 201 len 32768
158712024-01-24T19:12:19.615Z05/30 Read block 412 len 40960
158722024-01-24T19:12:19.615Z06/30 Write block 280 len 12288 data: 16 16 17
158732024-01-24T19:12:19.616Z07/30 Write block 347 len 32768 data: 11 9 12 10 11 10 11 10
158742024-01-24T19:12:19.616Z08/30 Read block 519 len 12288
158752024-01-24T19:12:19.616Z09/30 Write block 39 len 28672 data: 6 6 8 7 8 11 14
158762024-01-24T19:12:19.616Z10/30 Read block 543 len 28672
158772024-01-24T19:12:19.616Z11/30 Read block 84 len 32768
158782024-01-24T19:12:19.616Z12/30 Read block 241 len 4096
158792024-01-24T19:12:19.616Z13/30 Flush
158802024-01-24T19:12:19.616Z14/30 Write block 260 len 8192 data: 11 11
158812024-01-24T19:12:19.616Z15/30 Read block 105 len 36864
158822024-01-24T19:12:19.616Z16/30 Flush
158832024-01-24T19:12:19.616Z17/30 Write block 172 len 12288 data: 17 14 17
158842024-01-24T19:12:19.616Z18/30 Read block 356 len 12288
158852024-01-24T19:12:19.616Z19/30 Write block 540 len 28672 data: 10 10 14 13 15 13 14
158862024-01-24T19:12:19.616Z20/30 Read block 448 len 16384
158872024-01-24T19:12:19.616Z21/30 Read block 119 len 16384
158882024-01-24T19:12:19.616Z22/30 Read block 578 len 8192
158892024-01-24T19:12:19.616Z23/30 Read block 140 len 20480
158902024-01-24T19:12:19.617Z24/30 Write block 583 len 16384 data: 11 12 11 12
158912024-01-24T19:12:19.617Z25/30 Read block 178 len 28672
158922024-01-24T19:12:19.617Z26/30 Read block 151 len 12288
158932024-01-24T19:12:19.617Z27/30 Write block 480 len 40960 data: 11 12 12 9 9 9 10 7 9 9
158942024-01-24T19:12:19.617Z28/30 Read block 250 len 20480
158952024-01-24T19:12:19.617Z29/30 Write block 555 len 40960 data: 8 10 12 12 12 9 12 9 10 9
158962024-01-24T19:12:19.617Z30/30 Read block 137 len 32768
158972024-01-24T19:12:19.617Z----------------------------------------------------------------
158982024-01-24T19:12:19.617Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
158992024-01-24T19:12:19.617ZStates: Active Active Active
159002024-01-24T19:12:19.617ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159012024-01-24T19:12:19.617Z 1 Acked 1000 Write 7 Done Sent Done false
159022024-01-24T19:12:19.617Z 2 Acked 1001 Write 3 Done Sent Done false
159032024-01-24T19:12:19.617Z 3 Acked 1002 Read 8 Done Sent Done false
159042024-01-24T19:12:19.617Z 4 Acked 1003 Read 8 Done Sent Done false
159052024-01-24T19:12:19.617Z 5 Acked 1004 Read 10 Done Sent Done false
159062024-01-24T19:12:19.617Z 6 Acked 1005 Write 3 Done Sent Done false
159072024-01-24T19:12:19.618Z 7 Acked 1006 Write 8 Done Sent Done false
159082024-01-24T19:12:19.618Z 8 Acked 1007 Read 3 Done Sent Done false
159092024-01-24T19:12:19.618Z 9 Acked 1008 Write 7 Done Sent Done false
159102024-01-24T19:12:19.618Z 10 Acked 1009 Read 7 Done Sent Done false
159112024-01-24T19:12:19.618Z 11 Acked 1010 Read 8 Done Sent Done false
159122024-01-24T19:12:19.618Z 12 Acked 1011 Read 1 Done Sent Done false
159132024-01-24T19:12:19.618Z 13 Acked 1012 Flush 0 Done Sent Done false
159142024-01-24T19:12:19.618Z 14 Acked 1013 Write 2 Done Sent Done false
159152024-01-24T19:12:19.618Z 15 Acked 1014 Read 9 Done Sent Done false
159162024-01-24T19:12:19.618Z 16 Acked 1015 Flush 0 Done Sent Done false
159172024-01-24T19:12:19.618Z 17 Acked 1016 Write 3 Done Sent Done false
159182024-01-24T19:12:19.618Z 18 Acked 1017 Read 3 Done Sent Done false
159192024-01-24T19:12:19.618Z 19 Acked 1018 Write 7 Done Sent Done false
159202024-01-24T19:12:19.618Z 20 Acked 1019 Read 4 Done Sent Done false
159212024-01-24T19:12:19.618Z 21 Acked 1020 Read 4 Done Sent Done false
159222024-01-24T19:12:19.618Z 22 Acked 1021 Read 2 Done Sent Done false
159232024-01-24T19:12:19.618Z 23 Acked 1022 Read 5 Done Sent Done false
159242024-01-24T19:12:19.619Z 24 Acked 1023 Write 4 Done Sent Done false
159252024-01-24T19:12:19.682Z 25 Acked 1024 Read 7 Done Sent Done false
159262024-01-24T19:12:19.682Z 26 Acked 1025 Read 3 Done Sent Done false
159272024-01-24T19:12:19.682Z 27 Acked 1026 Write 10 Done Sent Done false
159282024-01-24T19:12:19.682Z 28 Acked 1027 Read 5 Done Sent Done false
159292024-01-24T19:12:19.682Z 29 Acked 1028 Write 10 Done Sent Done false
159302024-01-24T19:12:19.682Z 30 Acked 1029 Read 8 Done Sent Sent false
159312024-01-24T19:12:19.683Z STATES DS:0 DS:1 DS:2 TOTAL
159322024-01-24T19:12:19.683Z New 0 0 0 0
159332024-01-24T19:12:19.683Z Sent 0 30 1 31
159342024-01-24T19:12:19.683Z Done 30 0 29 59
159352024-01-24T19:12:19.683Z Skipped 0 0 0 0
159362024-01-24T19:12:19.683Z Error 0 0 0 0
159372024-01-24T19:12:19.683ZLast Flush: 1015 0 1015
159382024-01-24T19:12:19.683ZDownstairs last five completed:
159392024-01-24T19:12:19.683ZUpstairs last five completed: 30 29 28 27 26
159402024-01-24T19:12:19.683ZWrote out file "/tmp/test_repair_verify.data"
159412024-01-24T19:12:19.683Z
159422024-01-24T19:12:19.683ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159432024-01-24T19:12:19.683ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159442024-01-24T19:12:19.683Z 0 000-019 139 139 139 244 244 244 F F F
159452024-01-24T19:12:19.683Z 1 020-039 148 144 148 257 250 257 F F F <---
159462024-01-24T19:12:19.683Z 2 040-059 148 139 148 257 244 257 F F F <---
159472024-01-24T19:12:19.683Z 3 060-079 142 142 142 247 247 247 F F F
159482024-01-24T19:12:19.683Z 4 080-099 144 144 144 251 251 251 F F F
159492024-01-24T19:12:19.684Z 5 100-119 148 144 148 257 251 257 F F F <---
159502024-01-24T19:12:19.684Z 6 120-139 145 145 145 252 252 252 F F F
159512024-01-24T19:12:19.684Z 7 140-159 147 147 147 256 256 256 F F F
159522024-01-24T19:12:19.684Z 8 160-179 146 146 146 255 255 255 T F T <---
159532024-01-24T19:12:19.684Z 9 180-199 146 146 146 255 255 255 F F F
159542024-01-24T19:12:19.684Z 10 200-219 143 143 143 248 248 248 F F F
159552024-01-24T19:12:19.684Z 11 220-239 144 144 144 250 250 250 F F F
159562024-01-24T19:12:19.684Z 12 240-259 147 147 147 256 256 256 F F F
159572024-01-24T19:12:19.684Z 13 260-279 148 146 148 258 254 258 F F F <---
159582024-01-24T19:12:19.684Z 14 280-299 148 145 148 257 252 257 F F F <---
159592024-01-24T19:12:19.684Z 15 300-319 142 142 142 247 247 247 F F F
159602024-01-24T19:12:19.684Z 16 320-339 147 147 147 256 256 256 F F F
159612024-01-24T19:12:19.684Z 17 340-359 148 146 148 257 255 257 F F F <---
159622024-01-24T19:12:19.684Z 18 360-379 146 146 146 254 254 254 F F F
159632024-01-24T19:12:19.684Z 19 380-399 148 145 148 257 252 257 F F F <---
159642024-01-24T19:12:19.684Z 20 400-419 146 146 146 255 255 255 F F F
159652024-01-24T19:12:19.684Z 21 420-439 143 143 143 248 248 248 F F F
159662024-01-24T19:12:19.684Z 22 440-459 140 140 140 245 245 245 F F F
159672024-01-24T19:12:19.685Z 23 460-479 129 129 129 222 222 222 F F F
159682024-01-24T19:12:19.928Z 24 480-499 146 146 146 255 255 255 T F T <---
159692024-01-24T19:12:19.928Z 25 500-519 147 147 147 256 256 256 F F F
159702024-01-24T19:12:19.928Z 26 520-539 144 144 144 249 249 249 F F F
159712024-01-24T19:12:19.928Z 27 540-559 144 144 144 250 250 250 T F T <---
159722024-01-24T19:12:19.929Z 28 560-579 140 140 140 245 245 245 T F T <---
159732024-01-24T19:12:19.991Z 29 580-599 146 146 146 255 255 255 T F T <---
159742024-01-24T19:12:19.991ZMax gen: 148, Max flush: 258
159752024-01-24T19:12:19.991ZError: Difference in extent metadata found!
159762024-01-24T19:12:19.992ZA Difference in extent metadata is expected here
159772024-01-24T19:12:19.992ZOn loop 74
159782024-01-24T19:12:19.992Z
159792024-01-24T19:12:19.992ZVerifying data now
159802024-01-24T19:12:24.694ZNo change after verify
159812024-01-24T19:12:24.694ZLoop: 74 Downstairs dump after verify (and repair):
159822024-01-24T19:12:25.755ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159832024-01-24T19:12:25.755Z 0 000-019 139 139 139 244 244 244 F F F
159842024-01-24T19:12:25.755Z 1 020-039 148 148 148 257 257 257 F F F
159852024-01-24T19:12:25.755Z 2 040-059 148 148 148 257 257 257 F F F
159862024-01-24T19:12:25.755Z 3 060-079 142 142 142 247 247 247 F F F
159872024-01-24T19:12:25.755Z 4 080-099 144 144 144 251 251 251 F F F
159882024-01-24T19:12:25.755Z 5 100-119 148 148 148 257 257 257 F F F
159892024-01-24T19:12:25.756Z 6 120-139 145 145 145 252 252 252 F F F
159902024-01-24T19:12:25.756Z 7 140-159 147 147 147 256 256 256 F F F
159912024-01-24T19:12:25.756Z 8 160-179 149 149 149 259 259 259 F F F
159922024-01-24T19:12:25.756Z 9 180-199 146 146 146 255 255 255 F F F
159932024-01-24T19:12:25.756Z 10 200-219 143 143 143 248 248 248 F F F
159942024-01-24T19:12:25.756Z 11 220-239 144 144 144 250 250 250 F F F
159952024-01-24T19:12:25.756Z 12 240-259 147 147 147 256 256 256 F F F
159962024-01-24T19:12:25.756Z 13 260-279 148 148 148 258 258 258 F F F
159972024-01-24T19:12:25.756Z 14 280-299 148 148 148 257 257 257 F F F
159982024-01-24T19:12:25.798Z 15 300-319 142 142 142 247 247 247 F F F
159992024-01-24T19:12:25.798Z 16 320-339 147 147 147 256 256 256 F F F
160002024-01-24T19:12:25.798Z 17 340-359 148 148 148 257 257 257 F F F
160012024-01-24T19:12:25.798Z 18 360-379 146 146 146 254 254 254 F F F
160022024-01-24T19:12:25.798Z 19 380-399 148 148 148 257 257 257 F F F
160032024-01-24T19:12:25.799Z 20 400-419 146 146 146 255 255 255 F F F
160042024-01-24T19:12:25.799Z 21 420-439 143 143 143 248 248 248 F F F
160052024-01-24T19:12:25.799Z 22 440-459 140 140 140 245 245 245 F F F
160062024-01-24T19:12:25.799Z 23 460-479 129 129 129 222 222 222 F F F
160072024-01-24T19:12:25.799Z 24 480-499 149 149 149 259 259 259 F F F
160082024-01-24T19:12:25.799Z 25 500-519 147 147 147 256 256 256 F F F
160092024-01-24T19:12:25.799Z 26 520-539 144 144 144 249 249 249 F F F
160102024-01-24T19:12:25.799Z 27 540-559 149 149 149 259 259 259 F F F
160112024-01-24T19:12:25.799Z 28 560-579 149 149 149 259 259 259 F F F
160122024-01-24T19:12:25.799Z 29 580-599 149 149 149 259 259 259 F F F
160132024-01-24T19:12:25.799ZMax gen: 149, Max flush: 259
160142024-01-24T19:12:25.799Z
160152024-01-24T19:12:25.799ZBegin loop 75
160162024-01-24T19:12:25.800ZDownstairs to restart: 1
160172024-01-24T19:12:25.800Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.260626285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729}
160182024-01-24T19:12:25.800Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261123232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729}
160192024-01-24T19:12:25.850Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261193057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729}
160202024-01-24T19:12:25.850Z{"msg":"Crucible f11c712e-a3ce-4654-987d-cc465d0fb63d has session id: 628cf896-2c30-4163-9346-2f53131e96d9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261234683Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160212024-01-24T19:12:25.850Z{"msg":"Upstairs opts: Upstairs UUID: f11c712e-a3ce-4654-987d-cc465d0fb63d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261308662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160222024-01-24T19:12:25.850Z{"msg":"Crucible stats registered with UUID: f11c712e-a3ce-4654-987d-cc465d0fb63d","v":0,"name":"crucible","level":30Crucible runtime is spawned
160232024-01-24T19:12:25.850Z,"time":"2024-01-24T19:12:22.261406574Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160242024-01-24T19:12:25.850Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261773656Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729}
160252024-01-24T19:12:25.850Z{"msg":"f11c712e-a3ce-4654-987d-cc465d0fb63d active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261887756Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160262024-01-24T19:12:25.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261945631Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160272024-01-24T19:12:25.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.261970399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160282024-01-24T19:12:25.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.26204798Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160292024-01-24T19:12:25.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262095901Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160302024-01-24T19:12:25.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262137917Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160312024-01-24T19:12:25.851Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262179234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160322024-01-24T19:12:25.851Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262261828Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160332024-01-24T19:12:25.851Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262333094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160342024-01-24T19:12:25.851Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.26237704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160352024-01-24T19:12:25.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262708034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160362024-01-24T19:12:25.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262785527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160372024-01-24T19:12:25.851Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.262922443Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160382024-01-24T19:12:25.851Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264025499Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160392024-01-24T19:12:25.864Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264092479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160402024-01-24T19:12:25.864Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264265851Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160412024-01-24T19:12:25.864Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264370039Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160422024-01-24T19:12:25.864Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264414988Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160432024-01-24T19:12:25.864Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.264453696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160442024-01-24T19:12:25.864Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.265516635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160452024-01-24T19:12:25.864Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.265584924Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160462024-01-24T19:12:25.864Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.265873232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160472024-01-24T19:12:25.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.266600455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160482024-01-24T19:12:25.864Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.266694016Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160492024-01-24T19:12:25.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.266805003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160502024-01-24T19:12:25.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.266852441Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160512024-01-24T19:12:25.880Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.266931634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160522024-01-24T19:12:25.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267006956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160532024-01-24T19:12:25.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267078239Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160542024-01-24T19:12:25.880Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267153939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160552024-01-24T19:12:25.880Z{"msg":"[0]R flush_numbers[0..12]: [244, 257, 257, 247, 251, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267228378Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160562024-01-24T19:12:25.880Z{"msg":"[0]R generation[0..12]: [139, 148, 148, 142, 144, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
160572024-01-24T19:12:25.880Z,"time":"2024-01-24T19:12:22.267302491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160582024-01-24T19:12:25.880Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267387108Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160592024-01-24T19:12:25.881Z{"msg":"[1]R flush_numbers[0..12]: [244, 257, 257, 247, 251, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267464955Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160602024-01-24T19:12:25.881Z{"msg":"[1]R generation[0..12]: [139, 148, 148, 142, 144, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267546606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160612024-01-24T19:12:25.881Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267629753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160622024-01-24T19:12:25.881Z{"msg":"[2]R flush_numbers[0..12]: [244, 257, 257, 247, 251, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267712638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160632024-01-24T19:12:25.881Z{"msg":"[2]R generation[0..12]: [139, 148, 148, 142, 144, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267795334Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160642024-01-24T19:12:25.881Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
160652024-01-24T19:12:25.881Z,"time":"2024-01-24T19:12:22.267879165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160662024-01-24T19:12:25.881Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.267957293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160672024-01-24T19:12:25.881Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268011106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160682024-01-24T19:12:25.881Z{"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268036312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160692024-01-24T19:12:25.897Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268079654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160702024-01-24T19:12:25.897Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.2681052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160712024-01-24T19:12:25.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268147055Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160722024-01-24T19:12:25.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.26817217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160732024-01-24T19:12:25.897Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268216343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160742024-01-24T19:12:25.897Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268241664Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160752024-01-24T19:12:25.897Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268283866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160762024-01-24T19:12:25.898Z{"msg":"f11c712e-a3ce-4654-987d-cc465d0fb63d is now active with session: 628cf896-2c30-4163-9346-2f53131e96d9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268308688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160772024-01-24T19:12:25.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268349791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729,"session_id":"628cf896-2c30-4163-9346-2f53131e96d9"}
160782024-01-24T19:12:25.898Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:22.268373699Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1729}
160792024-01-24T19:12:25.898ZLoading write count information from file "/tmp/test_repair_verify.data"
160802024-01-24T19:12:25.910ZRun Repair workload
160812024-01-24T19:12:25.910Z01/30 Write block 11 len 16384 data: 7 6 6 6
160822024-01-24T19:12:26.419Z02/30 Read block 56 len 16384
160832024-01-24T19:12:26.419Z03/30 Read block 266 len 36864
160842024-01-24T19:12:26.419Z04/30 Flush
160852024-01-24T19:12:26.420Z05/30 Write block 263 len 20480 data: 9 11 12 10 11
160862024-01-24T19:12:26.421Z06/30 Read block 273 len 12288
160872024-01-24T19:12:26.421Z07/30 Read block 459 len 4096
160882024-01-24T19:12:26.421Z08/30 Write block 64 len 36864 data: 13 10 8 8 5 5 6 5 7
160892024-01-24T19:12:26.421Z09/30 Write block 87 len 20480 data: 10 8 7 8 10
160902024-01-24T19:12:26.421Z10/30 Read block 461 len 12288
160912024-01-24T19:12:26.421Z11/30 Write block 18 len 4096 data: 6
160922024-01-24T19:12:26.421Z12/30 Read block 22 len 40960
160932024-01-24T19:12:26.421Z13/30 Read block 386 len 12288
160942024-01-24T19:12:26.421Z14/30 Flush
160952024-01-24T19:12:26.421Z15/30 Write block 569 len 40960 data: 17 11 11 13 10 10 11 11 6 8
160962024-01-24T19:12:26.421Z16/30 Read block 346 len 8192
160972024-01-24T19:12:26.421Z17/30 Write block 41 len 20480 data: 9 8 9 12 15
160982024-01-24T19:12:26.422Z18/30 Read block 464 len 40960
160992024-01-24T19:12:26.422Z19/30 Write block 351 len 32768 data: 12 11 12 11 12 13 11 13
161002024-01-24T19:12:26.422Z20/30 Read block 129 len 32768
161012024-01-24T19:12:26.422Z21/30 Read block 470 len 12288
161022024-01-24T19:12:26.422Z22/30 Write block 419 len 40960 data: 12 9 9 10 9 8 8 7 11 12
161032024-01-24T19:12:26.422Z23/30 Read block 570 len 8192
161042024-01-24T19:12:26.422Z24/30 Write block 462 len 20480 data: 8 7 7 9 9
161052024-01-24T19:12:26.422Z25/30 Read block 77 len 12288
161062024-01-24T19:12:26.422Z26/30 Read block 181 len 28672
161072024-01-24T19:12:26.422Z27/30 Read block 519 len 32768
161082024-01-24T19:12:26.422Z28/30 Read block 58 len 36864
161092024-01-24T19:12:26.422Z29/30 Read block 367 len 16384
161102024-01-24T19:12:26.422Z30/30 Read block 500 len 36864
161112024-01-24T19:12:26.422Z----------------------------------------------------------------
161122024-01-24T19:12:26.424Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161132024-01-24T19:12:26.424ZStates: Active Active Active
161142024-01-24T19:12:26.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161152024-01-24T19:12:26.424Z 1 Acked 1000 Write 4 Done Done Done false
161162024-01-24T19:12:26.424Z 2 Acked 1001 Read 4 Done Done Done false
161172024-01-24T19:12:26.424Z 3 Acked 1002 Read 9 Done Sent Done false
161182024-01-24T19:12:26.424Z 4 Acked 1003 Flush 0 Done Sent Done false
161192024-01-24T19:12:26.424Z 5 Acked 1004 Write 5 Done Sent Done false
161202024-01-24T19:12:26.424Z 6 Acked 1005 Read 3 Done Sent Done false
161212024-01-24T19:12:26.424Z 7 Acked 1006 Read 1 Done Sent Done false
161222024-01-24T19:12:26.424Z 8 Acked 1007 Write 9 Done Sent Done false
161232024-01-24T19:12:26.424Z 9 Acked 1008 Write 5 Done Sent Done false
161242024-01-24T19:12:26.425Z 10 Acked 1009 Read 3 Done Sent Done false
161252024-01-24T19:12:26.425Z 11 Acked 1010 Write 1 Done Sent Done false
161262024-01-24T19:12:26.425Z 12 Acked 1011 Read 10 Done Sent Done false
161272024-01-24T19:12:26.425Z 13 Acked 1012 Read 3 Done Sent Done false
161282024-01-24T19:12:26.425Z 14 Acked 1013 Flush 0 Done Sent Done false
161292024-01-24T19:12:26.425Z 15 Acked 1014 Write 10 Done Sent Done false
161302024-01-24T19:12:26.425Z 16 Acked 1015 Read 2 Done Sent Done false
161312024-01-24T19:12:26.425Z 17 Acked 1016 Write 5 Done Sent Done false
161322024-01-24T19:12:26.425Z 18 Acked 1017 Read 10 Done Sent Done false
161332024-01-24T19:12:26.425Z 19 Acked 1018 Write 8 Done Sent Done false
161342024-01-24T19:12:26.425Z 20 Acked 1019 Read 8 Done Sent Done false
161352024-01-24T19:12:26.432Z 21 Acked 1020 Read 3 Done Sent Done false
161362024-01-24T19:12:26.433Z 22 Acked 1021 Write 10 Done Sent Done false
161372024-01-24T19:12:26.433Z 23 Acked 1022 Read 2 Done Sent Done false
161382024-01-24T19:12:26.433Z 24 Acked 1023 Write 5 Done Sent Done false
161392024-01-24T19:12:26.433Z 25 Acked 1024 Read 3 Done Sent Done false
161402024-01-24T19:12:26.433Z 26 Acked 1025 Read 7 Done Sent Done false
161412024-01-24T19:12:26.433Z 27 Acked 1026 Read 8 Done Sent Done false
161422024-01-24T19:12:26.433Z 28 Acked 1027 Read 9 Done Sent Done false
161432024-01-24T19:12:26.433Z 29 Acked 1028 Read 4 Done Sent Done false
161442024-01-24T19:12:26.433Z 30 Acked 1029 Read 9 Done Sent Sent false
161452024-01-24T19:12:26.433Z STATES DS:0 DS:1 DS:2 TOTAL
161462024-01-24T19:12:26.433Z New 0 0 0 0
161472024-01-24T19:12:26.433Z Sent 0 28 1 29
161482024-01-24T19:12:26.434Z Done 30 2 29 61
161492024-01-24T19:12:26.434Z Skipped 0 0 0 0
161502024-01-24T19:12:26.434Z Error 0 0 0 0
161512024-01-24T19:12:26.446ZLast Flush: 1013 0 1013
161522024-01-24T19:12:26.447ZDownstairs last five completed:
161532024-01-24T19:12:26.447ZUpstairs last five completed: 30 29 28 27 26
161542024-01-24T19:12:26.447ZWrote out file "/tmp/test_repair_verify.data"
161552024-01-24T19:12:26.447Z
161562024-01-24T19:12:26.447ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161572024-01-24T19:12:26.447ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161582024-01-24T19:12:26.447Z 0 000-019 150 139 150 261 244 261 F T F <---
161592024-01-24T19:12:26.447Z 1 020-039 148 148 148 257 257 257 F F F
161602024-01-24T19:12:26.447Z 2 040-059 148 148 148 257 257 257 T F T <---
161612024-01-24T19:12:26.447Z 3 060-079 150 142 150 261 247 261 F F F <---
161622024-01-24T19:12:26.447Z 4 080-099 150 144 150 261 251 261 F F F <---
161632024-01-24T19:12:26.447Z 5 100-119 148 148 148 257 257 257 F F F
161642024-01-24T19:12:26.447Z 6 120-139 145 145 145 252 252 252 F F F
161652024-01-24T19:12:26.447Z 7 140-159 147 147 147 256 256 256 F F F
161662024-01-24T19:12:26.448Z 8 160-179 149 149 149 259 259 259 F F F
161672024-01-24T19:12:26.448Z 9 180-199 146 146 146 255 255 255 F F F
161682024-01-24T19:12:26.448Z 10 200-219 143 143 143 248 248 248 F F F
161692024-01-24T19:12:26.448Z 11 220-239 144 144 144 250 250 250 F F F
161702024-01-24T19:12:26.448Z 12 240-259 147 147 147 256 256 256 F F F
161712024-01-24T19:12:26.448Z 13 260-279 150 148 150 261 258 261 F F F <---
161722024-01-24T19:12:26.448Z 14 280-299 148 148 148 257 257 257 F F F
161732024-01-24T19:12:26.448Z 15 300-319 142 142 142 247 247 247 F F F
161742024-01-24T19:12:26.448Z 16 320-339 147 147 147 256 256 256 F F F
161752024-01-24T19:12:26.448Z 17 340-359 148 148 148 257 257 257 T F T <---
161762024-01-24T19:12:26.448Z 18 360-379 146 146 146 254 254 254 F F F
161772024-01-24T19:12:26.448Z 19 380-399 148 148 148 257 257 257 F F F
161782024-01-24T19:12:26.448Z 20 400-419 146 146 146 255 255 255 T F T <---
161792024-01-24T19:12:26.448Z 21 420-439 143 143 143 248 248 248 T F T <---
161802024-01-24T19:12:26.448Z 22 440-459 140 140 140 245 245 245 F F F
161812024-01-24T19:12:26.448Z 23 460-479 129 129 129 222 222 222 T F T <---
161822024-01-24T19:12:48.493Z 24 480-499 149 149 149 259 259 259 F F F
161832024-01-24T19:12:48.493Z 25 500-519 147 147 147 256 256 256 F F F
161842024-01-24T19:12:48.494Z 26 520-539 144 144 144 249 249 249 F F F
161852024-01-24T19:12:48.496Z 27 540-559 149 149 149 259 259 259 F F F
161862024-01-24T19:12:48.496Z 28 560-579 149 149 149 259 259 259 T F T <---
161872024-01-24T19:12:48.496Z 29 580-599 149 149 149 259 259 259 F F F
161882024-01-24T19:12:48.496ZMax gen: 150, Max flush: 261
161892024-01-24T19:12:48.496ZError: Difference in extent metadata found!
161902024-01-24T19:12:48.496ZA Difference in extent metadata is expected here
161912024-01-24T19:12:48.497ZOn loop 75
161922024-01-24T19:12:48.497Z
161932024-01-24T19:12:48.497ZVerifying data now
161942024-01-24T19:12:48.497ZNo change after verify
161952024-01-24T19:12:48.497ZLoop: 75 Downstairs dump after verify (and repair):
161962024-01-24T19:12:48.497ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161972024-01-24T19:12:48.497Z 0 000-019 150 150 150 261 261 261 F F F
161982024-01-24T19:12:48.497Z 1 020-039 148 148 148 257 257 257 F F F
161992024-01-24T19:12:48.497Z 2 040-059 151 151 151 262 262 262 F F F
162002024-01-24T19:12:48.497Z 3 060-079 150 150 150 261 261 261 F F F
162012024-01-24T19:12:48.497Z 4 080-099 150 150 150 261 261 261 F F F
162022024-01-24T19:12:48.498Z 5 100-119 148 148 148 257 257 257 F F F
162032024-01-24T19:12:48.498Z 6 120-139 145 145 145 252 252 252 F F F
162042024-01-24T19:12:48.498Z 7 140-159 147 147 147 256 256 256 F F F
162052024-01-24T19:12:48.498Z 8 160-179 149 149 149 259 259 259 F F F
162062024-01-24T19:12:48.498Z 9 180-199 146 146 146 255 255 255 F F F
162072024-01-24T19:12:48.499Z 10 200-219 143 143 143 248 248 248 F F F
162082024-01-24T19:12:48.499Z 11 220-239 144 144 144 250 250 250 F F F
162092024-01-24T19:12:48.499Z 12 240-259 147 147 147 256 256 256 F F F
162102024-01-24T19:12:48.499Z 13 260-279 150 150 150 261 261 261 F F F
162112024-01-24T19:12:48.499Z 14 280-299 148 148 148 257 257 257 F F F
162122024-01-24T19:12:48.499Z 15 300-319 142 142 142 247 247 247 F F F
162132024-01-24T19:12:48.499Z 16 320-339 147 147 147 256 256 256 F F F
162142024-01-24T19:12:48.499Z 17 340-359 151 151 151 262 262 262 F F F
162152024-01-24T19:12:48.499Z 18 360-379 146 146 146 254 254 254 F F F
162162024-01-24T19:12:48.499Z 19 380-399 148 148 148 257 257 257 F F F
162172024-01-24T19:12:48.499Z 20 400-419 151 151 151 262 262 262 F F F
162182024-01-24T19:12:48.499Z 21 420-439 151 151 151 262 262 262 F F F
162192024-01-24T19:12:48.499Z 22 440-459 140 140 140 245 245 245 F F F
162202024-01-24T19:12:48.499Z 23 460-479 151 151 151 262 262 262 F F F
162212024-01-24T19:12:48.499Z 24 480-499 149 149 149 259 259 259 F F F
162222024-01-24T19:12:48.499Z 25 500-519 147 147 147 256 256 256 F F F
162232024-01-24T19:12:48.499Z 26 520-539 144 144 144 249 249 249 F F F
162242024-01-24T19:12:48.499Z 27 540-559 149 149 149 259 259 259 F F F
162252024-01-24T19:12:48.499Z 28 560-579 151 151 151 262 262 262 F F F
162262024-01-24T19:12:48.499Z 29 580-599 149 149 149 259 259 259 F F F
162272024-01-24T19:12:48.499ZMax gen: 151, Max flush: 262
162282024-01-24T19:12:48.500Z
162292024-01-24T19:12:48.500ZBegin loop 76
162302024-01-24T19:12:48.500ZDownstairs to restart: 2
162312024-01-24T19:12:48.500Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.363280422Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737}
162322024-01-24T19:12:48.500Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.363759653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737}
162332024-01-24T19:12:48.500Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.363790134Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737}
162342024-01-24T19:12:48.500Z{"msg":"Crucible b149ba18-73a5-497c-966d-ebbdebe8b342 has session id: 04c753fa-162a-4701-99e3-717ecedcd62e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.363852645Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162352024-01-24T19:12:48.500Z{"msg":"Upstairs opts: Upstairs UUID: b149ba18-73a5-497c-966d-ebbdebe8b342, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.363944054Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"Crucible runtime is spawned
162362024-01-24T19:12:48.500Z04c753fa-162a-4701-99e3-717ecedcd62e"}
162372024-01-24T19:12:48.500Z{"msg":"Crucible stats registered with UUID: b149ba18-73a5-497c-966d-ebbdebe8b342","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364152347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162382024-01-24T19:12:48.500Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364227474Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737}
162392024-01-24T19:12:48.500Z{"msg":"b149ba18-73a5-497c-966d-ebbdebe8b342 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364437143Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162402024-01-24T19:12:48.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364497749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162412024-01-24T19:12:48.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364585738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162422024-01-24T19:12:48.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364650803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162432024-01-24T19:12:48.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364679293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162442024-01-24T19:12:48.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364709048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162452024-01-24T19:12:48.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364771529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162462024-01-24T19:12:48.501Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364799529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162472024-01-24T19:12:48.505Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.36483666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162482024-01-24T19:12:48.505Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.364875885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162492024-01-24T19:12:48.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.365105779Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162502024-01-24T19:12:48.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.365229964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162512024-01-24T19:12:48.513Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.365280006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"io task","client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162522024-01-24T19:12:48.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.36644813Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162532024-01-24T19:12:48.513Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.366531425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162542024-01-24T19:12:48.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.366577615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162552024-01-24T19:12:48.513Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.366661165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162562024-01-24T19:12:48.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.367398069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162572024-01-24T19:12:48.513Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.367465564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162582024-01-24T19:12:48.513Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.367632038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162592024-01-24T19:12:48.513Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.367700538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162602024-01-24T19:12:48.513Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368541861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162612024-01-24T19:12:48.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368623201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162622024-01-24T19:12:48.514Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368675317Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162632024-01-24T19:12:48.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368701758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162642024-01-24T19:12:48.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368744588Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162652024-01-24T19:12:48.514Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368770926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162662024-01-24T19:12:48.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.368794979Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162672024-01-24T19:12:48.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.369430653Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162682024-01-24T19:12:48.514Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.369588394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162692024-01-24T19:12:48.514Z{"msg":"[0]R flush_numbers[0..12]: [261, 257, 262, 261, 261, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.369679757Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162702024-01-24T19:12:48.514Z{"msg":"[0]R generation[0..12]: [150, 148, 151, 150, 150, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
162712024-01-24T19:12:48.515Z,"time":"2024-01-24T19:12:27.369850843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162722024-01-24T19:12:48.515Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370009177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162732024-01-24T19:12:48.515Z{"msg":"[1]R flush_numbers[0..12]: [261, 257, 262, 261, 261, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370062852Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162742024-01-24T19:12:48.515Z{"msg":"[1]R generation[0..12]: [150, 148, 151, 150, 150, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370140255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162752024-01-24T19:12:48.515Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370190369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162762024-01-24T19:12:48.515Z{"msg":"[2]R flush_numbers[0..12]: [261, 257, 262, 261, 261, 257, 252, 256, 259, 255, 248, 250]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.37025987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162772024-01-24T19:12:48.515Z{"msg":"[2]R generation[0..12]: [150, 148, 151, 150, 150, 148, 145, 147, 149, 146, 143, 144]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370338131Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162782024-01-24T19:12:48.516Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370418384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162792024-01-24T19:12:48.516Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370497212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162802024-01-24T19:12:48.516Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370574224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162812024-01-24T19:12:48.516Z{"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370648418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162822024-01-24T19:12:48.751Z{"msg":"All extents match","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162832024-01-24T19:12:48.751Z,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370726103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162842024-01-24T19:12:48.752Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370798598Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162852024-01-24T19:12:48.752Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370862509Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162862024-01-24T19:12:48.752Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370919643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162872024-01-24T19:12:48.752Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370949714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162882024-01-24T19:12:48.752Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.370998369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162892024-01-24T19:12:48.752Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.371022654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162902024-01-24T19:12:48.752Z{"msg":"b149ba18-73a5-497c-966d-ebbdebe8b342 is now active with session: 04c753fa-162a-4701-99e3-717ecedcd62e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.371046933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162912024-01-24T19:12:48.752Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.371088063Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737,"session_id":"04c753fa-162a-4701-99e3-717ecedcd62e"}
162922024-01-24T19:12:48.752Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:27.371111803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1737}
162932024-01-24T19:12:48.752ZLoading write count information from file "/tmp/test_repair_verify.data"
162942024-01-24T19:12:48.752ZRun Repair workload
162952024-01-24T19:12:48.752Z01/30 Write block 149 len 20480 data: 15 13 14 16 14
162962024-01-24T19:12:48.752Z02/30 Write block 148 len 4096 data: 15
162972024-01-24T19:12:48.752Z03/30 Read block 379 len 16384
162982024-01-24T19:12:48.752Z04/30 Write block 41 len 12288 data: 10 9 10
162992024-01-24T19:12:48.752Z05/30 Write block 476 len 16384 data: 14 13 11 8
163002024-01-24T19:12:48.752Z06/30 Write block 499 len 24576 data: 14 11 12 12 8 12
163012024-01-24T19:12:48.752Z07/30 Read block 542 len 40960
163022024-01-24T19:12:48.752Z08/30 Write block 363 len 40960 data: 15 16 15 14 14 13 13 14 13 11
163032024-01-24T19:12:48.753Z09/30 Write block 487 len 40960 data: 8 10 10 10 9 10 11 12 11 15
163042024-01-24T19:12:48.753Z10/30 Write block 97 len 12288 data: 11 10 8
163052024-01-24T19:12:48.753Z11/30 Flush
163062024-01-24T19:12:48.753Z12/30 Write block 523 len 36864 data: 10 12 13 13 11 16 14 14 11
163072024-01-24T19:12:48.753Z13/30 Write block 567 len 28672 data: 16 18 18 12 12 14 11
163082024-01-24T19:12:48.753Z14/30 Read block 333 len 24576
163092024-01-24T19:12:48.753Z15/30 Read block 217 len 16384
163102024-01-24T19:12:48.753Z16/30 Read block 361 len 16384
163112024-01-24T19:12:48.753Z17/30 Write block 553 len 12288 data: 11 11 9
163122024-01-24T19:12:48.753Z18/30 Write block 335 len 28672 data: 11 11 11 11 11 11 10
163132024-01-24T19:12:48.753Z19/30 Write block 296 len 24576 data: 12 9 11 9 9 9
163142024-01-24T19:12:48.753Z20/30 Read block 531 len 32768
163152024-01-24T19:12:48.753Z21/30 Read block 561 len 24576
163162024-01-24T19:12:48.753Z22/30 Read block 131 len 20480
163172024-01-24T19:12:48.753Z23/30 Write block 209 len 36864 data: 10 12 10 10 9 6 6 9 8
163182024-01-24T19:12:48.753Z24/30 Write block 414 len 24576 data: 7 7 6 8 10 13
163192024-01-24T19:12:48.753Z25/30 Read block 252 len 20480
163202024-01-24T19:12:48.753Z26/30 Read block 486 len 36864
163212024-01-24T19:12:48.753Z27/30 Read block 43 len 4096
163222024-01-24T19:12:48.753Z28/30 Write block 222 len 4096 data: 10
163232024-01-24T19:12:48.754Z29/30 Read block 316 len 40960
163242024-01-24T19:12:48.754Z30/30 Read block 220 len 8192
163252024-01-24T19:12:48.754Z----------------------------------------------------------------
163262024-01-24T19:12:48.754Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
163272024-01-24T19:12:48.754ZStates: Active Active Active
163282024-01-24T19:12:48.754ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163292024-01-24T19:12:48.754Z 1 Acked 1000 Write 5 Done Done Done false
163302024-01-24T19:12:48.754Z 2 Acked 1001 Write 1 Done Done Done false
163312024-01-24T19:12:48.754Z 3 Acked 1002 Read 4 Done Done Done false
163322024-01-24T19:12:48.754Z 4 Acked 1003 Write 3 Done Done Sent false
163332024-01-24T19:12:48.754Z 5 Acked 1004 Write 4 Done Done Sent false
163342024-01-24T19:12:48.754Z 6 Acked 1005 Write 6 Done Done Sent false
163352024-01-24T19:12:48.754Z 7 Acked 1006 Read 10 Done Done Sent false
163362024-01-24T19:12:48.754Z 8 Acked 1007 Write 10 Done Done Sent false
163372024-01-24T19:12:48.757Z 9 Acked 1008 Write 10 Done Done Sent false
163382024-01-24T19:12:48.757Z 10 Acked 1009 Write 3 Done Done Sent false
163392024-01-24T19:12:48.757Z 11 Acked 1010 Flush 0 Done Done Sent false
163402024-01-24T19:12:48.757Z 12 Acked 1011 Write 9 Done Done Sent false
163412024-01-24T19:12:48.757Z 13 Acked 1012 Write 7 Done Done Sent false
163422024-01-24T19:12:48.757Z 14 Acked 1013 Read 6 Done Done Sent false
163432024-01-24T19:12:48.759Z 15 Acked 1014 Read 4 Done Done Sent false
163442024-01-24T19:12:48.759Z 16 Acked 1015 Read 4 Done Done Sent false
163452024-01-24T19:12:48.759Z 17 Acked 1016 Write 3 Done Done Sent false
163462024-01-24T19:12:48.759Z 18 Acked 1017 Write 7 Done Done Sent false
163472024-01-24T19:12:48.759Z 19 Acked 1018 Write 6 Done Done Sent false
163482024-01-24T19:12:48.759Z 20 Acked 1019 Read 8 Done Done Sent false
163492024-01-24T19:12:48.759Z 21 Acked 1020 Read 6 Done Done Sent false
163502024-01-24T19:12:48.759Z 22 Acked 1021 Read 5 Done Done Sent false
163512024-01-24T19:12:48.759Z 23 Acked 1022 Write 9 Done Done Sent false
163522024-01-24T19:12:48.759Z 24 Acked 1023 Write 6 Done Done Sent false
163532024-01-24T19:12:48.759Z 25 Acked 1024 Read 5 Done Done Sent false
163542024-01-24T19:12:48.759Z 26 Acked 1025 Read 9 Done Done Sent false
163552024-01-24T19:12:48.760Z 27 Acked 1026 Read 1 Done Done Sent false
163562024-01-24T19:12:48.760Z 28 Acked 1027 Write 1 Done Done Sent false
163572024-01-24T19:12:48.760Z 29 Acked 1028 Read 10 Done Done Sent false
163582024-01-24T19:12:48.760Z 30 Acked 1029 Read 2 Done Sent Sent false
163592024-01-24T19:12:48.760Z STATES DS:0 DS:1 DS:2 TOTAL
163602024-01-24T19:12:48.762Z New 0 0 0 0
163612024-01-24T19:12:48.762Z Sent 0 1 27 28
163622024-01-24T19:12:48.762Z Done 30 29 3 62
163632024-01-24T19:12:48.762Z Skipped 0 0 0 0
163642024-01-24T19:12:48.762Z Error 0 0 0 0
163652024-01-24T19:12:48.762ZLast Flush: 1010 1010 0
163662024-01-24T19:12:48.762ZDownstairs last five completed:
163672024-01-24T19:12:48.762ZUpstairs last five completed: 30 29 28 27 26
163682024-01-24T19:12:48.763ZWrote out file "/tmp/test_repair_verify.data"
163692024-01-24T19:12:48.763Z
163702024-01-24T19:12:48.763ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163712024-01-24T19:12:48.763ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163722024-01-24T19:12:48.763Z 0 000-019 150 150 150 261 261 261 F F F
163732024-01-24T19:12:48.763Z 1 020-039 148 148 148 257 257 257 F F F
163742024-01-24T19:12:48.763Z 2 040-059 152 152 151 263 263 262 F F F <---
163752024-01-24T19:12:48.763Z 3 060-079 150 150 150 261 261 261 F F F
163762024-01-24T19:12:48.763Z 4 080-099 152 152 150 263 263 261 F F F <---
163772024-01-24T19:12:48.763Z 5 100-119 148 148 148 257 257 257 F F F
163782024-01-24T19:12:48.763Z 6 120-139 145 145 145 252 252 252 F F F
163792024-01-24T19:12:48.763Z 7 140-159 152 152 147 263 263 256 F F T <---
163802024-01-24T19:12:48.763Z 8 160-179 149 149 149 259 259 259 F F F
163812024-01-24T19:12:48.763Z 9 180-199 146 146 146 255 255 255 F F F
163822024-01-24T19:12:48.919Z 10 200-219 143 143 143 248 248 248 T T F <---
163832024-01-24T19:12:48.919Z 11 220-239 144 144 144 250 250 250 T T F <---
163842024-01-24T19:12:48.919Z 12 240-259 147 147 147 256 256 256 F F F
163852024-01-24T19:12:48.932Z 13 260-279 150 150 150 261 261 261 F F F
163862024-01-24T19:12:48.932Z 14 280-299 148 148 148 257 257 257 T T F <---
163872024-01-24T19:12:48.933Z 15 300-319 142 142 142 247 247 247 T T F <---
163882024-01-24T19:12:48.933Z 16 320-339 147 147 147 256 256 256 T T F <---
163892024-01-24T19:12:48.933Z 17 340-359 151 151 151 262 262 262 T T F <---
163902024-01-24T19:12:48.933Z 18 360-379 152 152 146 263 263 254 F F F <---
163912024-01-24T19:12:48.933Z 19 380-399 148 148 148 257 257 257 F F F
163922024-01-24T19:12:48.933Z 20 400-419 151 151 151 262 262 262 T T F <---
163932024-01-24T19:12:48.933Z 21 420-439 151 151 151 262 262 262 F F F
163942024-01-24T19:12:48.933Z 22 440-459 140 140 140 245 245 245 F F F
163952024-01-24T19:12:48.933Z 23 460-479 152 152 151 263 263 262 F F F <---
163962024-01-24T19:12:48.933Z 24 480-499 152 152 149 263 263 259 F F F <---
163972024-01-24T19:12:48.933Z 25 500-519 152 152 147 263 263 256 F F F <---
163982024-01-24T19:12:48.933Z 26 520-539 144 144 144 249 249 249 T T F <---
163992024-01-24T19:12:48.934Z 27 540-559 149 149 149 259 259 259 T T F <---
164002024-01-24T19:12:48.934Z 28 560-579 151 151 151 262 262 262 T T F <---
164012024-01-24T19:12:48.934Z 29 580-599 149 149 149 259 259 259 F F F
164022024-01-24T19:12:48.934ZMax gen: 152, Max flush: 263
164032024-01-24T19:12:48.934ZError: Difference in extent metadata found!
164042024-01-24T19:12:48.934ZA Difference in extent metadata is expected here
164052024-01-24T19:12:48.934ZOn loop 76
164062024-01-24T19:12:48.934Z
164072024-01-24T19:12:48.934ZVerifying data now
164082024-01-24T19:12:48.934ZNo change after verify
164092024-01-24T19:12:48.934ZLoop: 76 Downstairs dump after verify (and repair):
164102024-01-24T19:12:48.938ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164112024-01-24T19:12:48.939Z 0 000-019 150 150 150 261 261 261 F F F
164122024-01-24T19:12:48.939Z 1 020-039 148 148 148 257 257 257 F F F
164132024-01-24T19:12:48.939Z 2 040-059 152 152 152 263 263 263 F F F
164142024-01-24T19:12:48.939Z 3 060-079 150 150 150 261 261 261 F F F
164152024-01-24T19:12:48.939Z 4 080-099 152 152 152 263 263 263 F F F
164162024-01-24T19:12:48.939Z 5 100-119 148 148 148 257 257 257 F F F
164172024-01-24T19:12:48.939Z 6 120-139 145 145 145 252 252 252 F F F
164182024-01-24T19:12:48.939Z 7 140-159 152 152 152 263 263 263 F F F
164192024-01-24T19:12:48.939Z 8 160-179 149 149 149 259 259 259 F F F
164202024-01-24T19:12:48.939Z 9 180-199 146 146 146 255 255 255 F F F
164212024-01-24T19:12:48.939Z 10 200-219 153 153 153 264 264 264 F F F
164222024-01-24T19:12:48.939Z 11 220-239 153 153 153 264 264 264 F F F
164232024-01-24T19:12:48.939Z 12 240-259 147 147 147 256 256 256 F F F
164242024-01-24T19:12:48.939Z 13 260-279 150 150 150 261 261 261 F F F
164252024-01-24T19:12:48.939Z 14 280-299 153 153 153 264 264 264 F F F
164262024-01-24T19:12:48.939Z 15 300-319 153 153 153 264 264 264 F F F
164272024-01-24T19:12:48.939Z 16 320-339 153 153 153 264 264 264 F F F
164282024-01-24T19:12:48.940Z 17 340-359 153 153 153 264 264 264 F F F
164292024-01-24T19:12:48.940Z 18 360-379 152 152 152 263 263 263 F F F
164302024-01-24T19:12:48.940Z 19 380-399 148 148 148 257 257 257 F F F
164312024-01-24T19:12:48.940Z 20 400-419 153 153 153 264 264 264 F F F
164322024-01-24T19:12:48.940Z 21 420-439 151 151 151 262 262 262 F F F
164332024-01-24T19:12:48.940Z 22 440-459 140 140 140 245 245 245 F F F
164342024-01-24T19:12:48.941Z 23 460-479 152 152 152 263 263 263 F F F
164352024-01-24T19:12:48.941Z 24 480-499 152 152 152 263 263 263 F F F
164362024-01-24T19:12:48.941Z 25 500-519 152 152 152 263 263 263 F F F
164372024-01-24T19:12:48.941Z 26 520-539 153 153 153 264 264 264 F F F
164382024-01-24T19:12:48.941Z 27 540-559 153 153 153 264 264 264 F F F
164392024-01-24T19:12:48.941Z 28 560-579 153 153 153 264 264 264 F F F
164402024-01-24T19:12:48.942Z 29 580-599 149 149 149 259 259 259 F F F
164412024-01-24T19:12:48.942ZMax gen: 153, Max flush: 264
164422024-01-24T19:12:48.942Z
164432024-01-24T19:12:48.942ZBegin loop 77
164442024-01-24T19:12:48.942ZDownstairs to restart: 1
164452024-01-24T19:12:48.942Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.984553771Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745}
164462024-01-24T19:12:48.942Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.984983138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745}
164472024-01-24T19:12:48.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985040224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745}
164482024-01-24T19:12:48.942Z{"msg":"Crucible 3d91d743-3a93-4532-95b8-18f9b6afef09 has session id: 6490690f-1fbe-4ba1-a924-d27d0fdb74d4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985061818Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164492024-01-24T19:12:48.942Z{"msg":"Upstairs opts: Upstairs UUID: 3d91d743-3a93-4532-95b8-18f9b6afef09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985125623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164502024-01-24T19:12:48.943Z{"msg":"Crucible stats registered with UUID: 3d91d743-3a93-4532-95b8-18f9b6afef09","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985180058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164512024-01-24T19:12:48.943ZCrucible runtime is spawned
164522024-01-24T19:12:48.943Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985481445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745}
164532024-01-24T19:12:48.943Z{"msg":"3d91d743-3a93-4532-95b8-18f9b6afef09 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985706587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164542024-01-24T19:12:48.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985791104Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164552024-01-24T19:12:48.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985859611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164562024-01-24T19:12:48.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.985956305Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164572024-01-24T19:12:48.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986029073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164582024-01-24T19:12:48.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986101037Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164592024-01-24T19:12:48.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986147199Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164602024-01-24T19:12:48.944Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986332913Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164612024-01-24T19:12:48.944Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986439091Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164622024-01-24T19:12:48.944Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.98654778Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164632024-01-24T19:12:48.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986625481Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164642024-01-24T19:12:48.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986699258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164652024-01-24T19:12:48.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.986773292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"io task","client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164662024-01-24T19:12:48.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.987783918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164672024-01-24T19:12:48.944Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.987851272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164682024-01-24T19:12:48.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.987907272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164692024-01-24T19:12:48.944Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.987988742Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164702024-01-24T19:12:48.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.988076462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164712024-01-24T19:12:48.944Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.988156391Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164722024-01-24T19:12:48.944Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.989153719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164732024-01-24T19:12:48.944Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.989222353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164742024-01-24T19:12:48.945Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.989349727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164752024-01-24T19:12:48.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990047674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164762024-01-24T19:12:48.945Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990112662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164772024-01-24T19:12:48.945Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990182267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164782024-01-24T19:12:48.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990273587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164792024-01-24T19:12:48.945Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990323861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164802024-01-24T19:12:48.947Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990369538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164812024-01-24T19:12:48.947Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990413034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164822024-01-24T19:12:49.162Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990457904Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164832024-01-24T19:12:49.162Z{"msg":"[0]R flush_numbers[0..12]: [261, 257, 263, 261, 263, 257, 252, 263, 259, 255, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.99050425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164842024-01-24T19:12:49.162Z{"msg":"[0]R generation[0..12]: [150, 148, 152, 150, 152, 148, 145, 152, 149, 146, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990551103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164852024-01-24T19:12:49.163Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990597942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164862024-01-24T19:12:49.163Z{"msg":"[1]R flush_numbers[0..12]: [261, 257, 263, 261, 263, 257, 252, 263, 259, 255, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990644584Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164872024-01-24T19:12:49.163Z{"msg":"[1]R generation[0..12]: [150, 148, 152, 150, 152, 148, 145, 152, 149, 146, 153, 153]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
164882024-01-24T19:12:49.163Z,"time":"2024-01-24T19:12:32.990691257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164892024-01-24T19:12:49.163Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990803241Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164902024-01-24T19:12:49.163Z{"msg":"[2]R flush_numbers[0..12]: [261, 257, 263, 261, 263, 257, 252, 263, 259, 255, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990855672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164912024-01-24T19:12:49.163Z{"msg":"[2]R generation[0..12]: [150, 148, 152, 150, 152, 148, 145, 152, 149, 146, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990910209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164922024-01-24T19:12:49.163Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.990957144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164932024-01-24T19:12:49.165Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991002838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164942024-01-24T19:12:49.165Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991049135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164952024-01-24T19:12:49.165Z{"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991101475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164962024-01-24T19:12:49.165Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991145737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164972024-01-24T19:12:49.165Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991191366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164982024-01-24T19:12:49.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991214254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
164992024-01-24T19:12:49.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991256473Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165002024-01-24T19:12:49.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991301591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165012024-01-24T19:12:49.166Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991347552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165022024-01-24T19:12:49.166ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165032024-01-24T19:12:49.166Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991394416Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165042024-01-24T19:12:49.167Z{"msg":"3d91d743-3a93-4532-95b8-18f9b6afef09 is now active with session: 6490690f-1fbe-4ba1-a924-d27d0fdb74d4","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991438956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165052024-01-24T19:12:49.167Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991481797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745,"session_id":"6490690f-1fbe-4ba1-a924-d27d0fdb74d4"}
165062024-01-24T19:12:49.167Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:32.991524661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1745}
165072024-01-24T19:12:49.167ZLoading write count information from file "/tmp/test_repair_verify.data"
165082024-01-24T19:12:49.167ZRun Repair workload
165092024-01-24T19:12:49.167ZError: Difference in extent metadata found!
165102024-01-24T19:12:49.167Z01/30 Write block 366 len 16384 data: 15 15 14 14
165112024-01-24T19:12:49.167Z02/30 Write block 502 len 16384 data: 13 9 13 11
165122024-01-24T19:12:49.167Z03/30 Write block 297 len 24576 data: 10 12 10 10 10 8
165132024-01-24T19:12:49.167Z04/30 Read block 178 len 40960
165142024-01-24T19:12:49.168Z05/30 Read block 219 len 12288
165152024-01-24T19:12:49.168Z06/30 Write block 387 len 8192 data: 8 9
165162024-01-24T19:12:49.168Z07/30 Read block 67 len 12288
165172024-01-24T19:12:49.168Z08/30 Write block 188 len 28672 data: 10 10 10 10 11 11 12
165182024-01-24T19:12:49.168Z09/30 Read block 248 len 16384
165192024-01-24T19:12:49.168Z10/30 Write block 423 len 12288 data: 10 9 9
165202024-01-24T19:12:49.168Z11/30 Write block 445 len 4096 data: 7
165212024-01-24T19:12:49.168Z12/30 Write block 422 len 40960 data: 11 11 10 10 8 12 13 15 16 15
165222024-01-24T19:12:49.168Z13/30 Read block 586 len 8192
165232024-01-24T19:12:49.168Z14/30 Read block 453 len 16384
165242024-01-24T19:12:49.169Z15/30 Read block 562 len 40960
165252024-01-24T19:12:49.169Z16/30 Write block 529 len 8192 data: 15 15
165262024-01-24T19:12:49.169Z17/30 Flush
165272024-01-24T19:12:49.169Z18/30 Write block 302 len 8192 data: 9 7
165282024-01-24T19:12:49.169Z19/30 Read block 428 len 28672
165292024-01-24T19:12:49.169Z20/30 Read block 469 len 28672
165302024-01-24T19:12:49.169Z21/30 Write block 22 len 8192 data: 11 10
165312024-01-24T19:12:49.169Z22/30 Write block 254 len 16384 data: 9 12 13 11
165322024-01-24T19:12:49.169Z23/30 Write block 280 len 32768 data: 17 17 18 16 12 13 12 12
165332024-01-24T19:12:49.169Z24/30 Write block 532 len 40960 data: 8 9 10 9 9 11 11 12 11 11
165342024-01-24T19:12:49.170Z25/30 Flush
165352024-01-24T19:12:49.170Z26/30 Flush
165362024-01-24T19:12:49.170Z27/30 Write block 461 len 8192 data: 8 9
165372024-01-24T19:12:49.171Z28/30 Write block 102 len 40960 data: 7 11 12 14 15 16 17 18 17 16
165382024-01-24T19:12:49.171Z29/30 Write block 457 len 28672 data: 9 9 7 7 9 10 8
165392024-01-24T19:12:49.171Z30/30 Write block 452 len 28672 data: 8 9 8 7 10 10 10
165402024-01-24T19:12:49.171Z----------------------------------------------------------------
165412024-01-24T19:12:49.171Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
165422024-01-24T19:12:49.171ZStates: Active Active Active
165432024-01-24T19:12:49.171ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165442024-01-24T19:12:49.171Z 1 Acked 1000 Write 4 Done Done Done false
165452024-01-24T19:12:49.171Z 2 Acked 1001 Write 4 Done Done Done false
165462024-01-24T19:12:49.171Z 3 Acked 1002 Write 6 Done Sent Done false
165472024-01-24T19:12:49.171Z 4 Acked 1003 Read 10 Done Sent Done false
165482024-01-24T19:12:49.171Z 5 Acked 1004 Read 3 Done Sent Done false
165492024-01-24T19:12:49.173Z 6 Acked 1005 Write 2 Done Sent Done false
165502024-01-24T19:12:49.173Z 7 Acked 1006 Read 3 Done Sent Done false
165512024-01-24T19:12:49.173Z 8 Acked 1007 Write 7 Done Sent Done false
165522024-01-24T19:12:49.173Z 9 Acked 1008 Read 4 Done Sent Done false
165532024-01-24T19:12:49.173Z 10 Acked 1009 Write 3 Done Sent Done false
165542024-01-24T19:12:49.173Z 11 Acked 1010 Write 1 Done Sent Done false
165552024-01-24T19:12:49.173Z 12 Acked 1011 Write 10 Done Sent Done false
165562024-01-24T19:12:49.174Z 13 Acked 1012 Read 2 Done Sent Done false
165572024-01-24T19:12:49.174Z 14 Acked 1013 Read 4 Done Sent Done false
165582024-01-24T19:12:49.174Z 15 Acked 1014 Read 10 Done Sent Done false
165592024-01-24T19:12:49.174Z 16 Acked 1015 Write 2 Done Sent Done false
165602024-01-24T19:12:49.174Z 17 Acked 1016 Flush 0 Done Sent Done false
165612024-01-24T19:12:49.174Z 18 Acked 1017 Write 2 Done Sent Done false
165622024-01-24T19:12:49.174Z 19 Acked 1018 Read 7 Done Sent Done false
165632024-01-24T19:12:49.174Z 20 Acked 1019 Read 7 Done Sent Done false
165642024-01-24T19:12:49.174Z 21 Acked 1020 Write 2 Done Sent Done false
165652024-01-24T19:12:49.174Z 22 Acked 1021 Write 4 Done Sent Done false
165662024-01-24T19:12:49.174Z 23 Acked 1022 Write 8 Done Sent Done false
165672024-01-24T19:12:49.174Z 24 Acked 1023 Write 10 Done Sent Done false
165682024-01-24T19:12:49.174Z 25 Acked 1024 Flush 0 Done Sent Done false
165692024-01-24T19:12:49.174Z 26 Acked 1025 Flush 0 Done Sent Done false
165702024-01-24T19:12:49.174Z 27 Acked 1026 Write 2 Done Sent Done false
165712024-01-24T19:12:49.174Z 28 Acked 1027 Write 10 Done Sent Done false
165722024-01-24T19:12:49.175Z 29 Acked 1028 Write 7 Done Sent Done false
165732024-01-24T19:12:49.175Z 30 Acked 1029 Write 7 Sent Sent Sent false
165742024-01-24T19:12:49.175Z STATES DS:0 DS:1 DS:2 TOTAL
165752024-01-24T19:12:49.175Z New 0 0 0 0
165762024-01-24T19:12:49.175Z Sent 1 28 1 30
165772024-01-24T19:12:49.175Z Done 29 2 29 60
165782024-01-24T19:12:49.175Z Skipped 0 0 0 0
165792024-01-24T19:12:49.175Z Error 0 0 0 0
165802024-01-24T19:12:49.175ZLast Flush: 1025 0 1025
165812024-01-24T19:12:49.175ZDownstairs last five completed:
165822024-01-24T19:12:49.475ZUpstairs last five completed: 30 29 28 27 26
165832024-01-24T19:12:49.475ZWrote out file "/tmp/test_repair_verify.data"
165842024-01-24T19:12:49.475Z
165852024-01-24T19:12:49.475ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
165862024-01-24T19:12:49.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165872024-01-24T19:12:49.477Z 0 000-019 150 150 150 261 261 261 F F F
165882024-01-24T19:12:49.477Z 1 020-039 154 148 154 266 257 266 F F F <---
165892024-01-24T19:12:49.477Z 2 040-059 152 152 152 263 263 263 F F F
165902024-01-24T19:12:49.477Z 3 060-079 150 150 150 261 261 261 F F F
165912024-01-24T19:12:49.477Z 4 080-099 152 152 152 263 263 263 F F F
165922024-01-24T19:12:49.477Z 5 100-119 148 148 148 257 257 257 T F T <---
165932024-01-24T19:12:49.478Z 6 120-139 145 145 145 252 252 252 F F F
165942024-01-24T19:12:49.478Z 7 140-159 152 152 152 263 263 263 F F F
165952024-01-24T19:12:49.478Z 8 160-179 149 149 149 259 259 259 F F F
165962024-01-24T19:12:49.478Z 9 180-199 154 146 154 265 255 265 F F F <---
165972024-01-24T19:12:49.478Z 10 200-219 153 153 153 264 264 264 F F F
165982024-01-24T19:12:49.478Z 11 220-239 153 153 153 264 264 264 F F F
165992024-01-24T19:12:49.478Z 12 240-259 154 147 154 266 256 266 F F F <---
166002024-01-24T19:12:49.478Z 13 260-279 150 150 150 261 261 261 F F F
166012024-01-24T19:12:49.478Z 14 280-299 154 153 154 266 264 266 F F F <---
166022024-01-24T19:12:49.478Z 15 300-319 154 153 154 266 264 266 F F F <---
166032024-01-24T19:12:49.478Z 16 320-339 153 153 153 264 264 264 F F F
166042024-01-24T19:12:49.478Z 17 340-359 153 153 153 264 264 264 F F F
166052024-01-24T19:12:49.478Z 18 360-379 154 152 154 265 263 265 F T F <---
166062024-01-24T19:12:49.479Z 19 380-399 154 148 154 265 257 265 F F F <---
166072024-01-24T19:12:49.479Z 20 400-419 153 153 153 264 264 264 F F F
166082024-01-24T19:12:49.479Z 21 420-439 154 151 154 265 262 265 F F F <---
166092024-01-24T19:12:49.479Z 22 440-459 154 140 154 265 245 265 T F T <---
166102024-01-24T19:12:49.486Z 23 460-479 152 152 152 263 263 263 T F T <---
166112024-01-24T19:12:49.486Z 24 480-499 152 152 152 263 263 263 F F F
166122024-01-24T19:12:49.486Z 25 500-519 154 152 154 265 263 265 F T F <---
166132024-01-24T19:12:49.486Z 26 520-539 154 153 154 266 264 266 F F F <---
166142024-01-24T19:12:49.486Z 27 540-559 154 153 154 266 264 266 F F F <---
166152024-01-24T19:12:49.486Z 28 560-579 153 153 153 264 264 264 F F F
166162024-01-24T19:12:49.487Z 29 580-599 149 149 149 259 259 259 F F F
166172024-01-24T19:12:49.487ZMax gen: 154, Max flush: 266
166182024-01-24T19:12:49.487ZA Difference in extent metadata is expected here
166192024-01-24T19:12:49.487ZOn loop 77
166202024-01-24T19:12:49.487Z
166212024-01-24T19:12:49.487ZVerifying data now
166222024-01-24T19:12:49.487ZNo change after verify
166232024-01-24T19:12:49.487ZLoop: 77 Downstairs dump after verify (and repair):
166242024-01-24T19:12:49.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166252024-01-24T19:12:49.487Z 0 000-019 150 150 150 261 261 261 F F F
166262024-01-24T19:12:49.487Z 1 020-039 154 154 154 266 266 266 F F F
166272024-01-24T19:12:49.487Z 2 040-059 152 152 152 263 263 263 F F F
166282024-01-24T19:12:49.488Z 3 060-079 150 150 150 261 261 261 F F F
166292024-01-24T19:12:49.488Z 4 080-099 152 152 152 263 263 263 F F F
166302024-01-24T19:12:49.488Z 5 100-119 155 155 155 267 267 267 F F F
166312024-01-24T19:12:49.488Z 6 120-139 145 145 145 252 252 252 F F F
166322024-01-24T19:12:49.489Z 7 140-159 152 152 152 263 263 263 F F F
166332024-01-24T19:12:49.489Z 8 160-179 149 149 149 259 259 259 F F F
166342024-01-24T19:12:49.489Z 9 180-199 154 154 154 265 265 265 F F F
166352024-01-24T19:12:49.491Z 10 200-219 153 153 153 264 264 264 F F F
166362024-01-24T19:12:49.492Z 11 220-239 153 153 153 264 264 264 F F F
166372024-01-24T19:12:49.492Z 12 240-259 154 154 154 266 266 266 F F F
166382024-01-24T19:12:49.492Z 13 260-279 150 150 150 261 261 261 F F F
166392024-01-24T19:12:49.492Z 14 280-299 154 154 154 266 266 266 F F F
166402024-01-24T19:12:49.492Z 15 300-319 154 154 154 266 266 266 F F F
166412024-01-24T19:12:49.492Z 16 320-339 153 153 153 264 264 264 F F F
166422024-01-24T19:12:49.492Z 17 340-359 153 153 153 264 264 264 F F F
166432024-01-24T19:12:49.492Z 18 360-379 154 154 154 265 265 265 F F F
166442024-01-24T19:12:49.492Z 19 380-399 154 154 154 265 265 265 F F F
166452024-01-24T19:12:49.492Z 20 400-419 153 153 153 264 264 264 F F F
166462024-01-24T19:12:49.492Z 21 420-439 154 154 154 265 265 265 F F F
166472024-01-24T19:12:49.492Z 22 440-459 155 155 155 267 267 267 F F F
166482024-01-24T19:12:49.492Z 23 460-479 155 155 155 267 267 267 F F F
166492024-01-24T19:12:49.492Z 24 480-499 152 152 152 263 263 263 F F F
166502024-01-24T19:12:49.493Z 25 500-519 154 154 154 265 265 265 F F F
166512024-01-24T19:12:49.493Z 26 520-539 154 154 154 266 266 266 F F F
166522024-01-24T19:12:49.493Z 27 540-559 154 154 154 266 266 266 F F F
166532024-01-24T19:12:49.493Z 28 560-579 153 153 153 264 264 264 F F F
166542024-01-24T19:12:49.493Z 29 580-599 149 149 149 259 259 259 F F F
166552024-01-24T19:12:49.493ZMax gen: 155, Max flush: 267
166562024-01-24T19:12:49.493Z
166572024-01-24T19:12:49.493ZBegin loop 78
166582024-01-24T19:12:49.493ZDownstairs to restart: 1
166592024-01-24T19:12:49.493Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.349759861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753}
166602024-01-24T19:12:49.493Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350166833Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753}
166612024-01-24T19:12:49.493Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350194947Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753}
166622024-01-24T19:12:49.493Z{"msg":"Crucible 2c9109e9-58b9-4cf5-affc-6c780067354b has session id: ec7adabf-1325-4e0e-a64e-429c9d1f0f9d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350221275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166632024-01-24T19:12:49.493Z{"msg":"Upstairs opts: Upstairs UUID: 2c9109e9-58b9-4cf5-affc-6c780067354b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350278986Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166642024-01-24T19:12:49.493Z{"msg":"Crucible stats registered with UUID: 2c9109e9-58b9-4cf5-affc-6c780067354b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350323576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166652024-01-24T19:12:49.493ZCrucible runtime is spawned
166662024-01-24T19:12:49.493Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350534464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753}
166672024-01-24T19:12:49.493Z{"msg":"2c9109e9-58b9-4cf5-affc-6c780067354b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350758081Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166682024-01-24T19:12:49.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350786855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166692024-01-24T19:12:49.493Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.35081101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166702024-01-24T19:12:49.493Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350833331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166712024-01-24T19:12:49.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350855932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166722024-01-24T19:12:49.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350877578Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166732024-01-24T19:12:49.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350904478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166742024-01-24T19:12:49.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350944384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166752024-01-24T19:12:49.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350971388Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166762024-01-24T19:12:49.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.350994107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166772024-01-24T19:12:49.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.351275461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166782024-01-24T19:12:49.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.351479046Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166792024-01-24T19:12:49.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.351534503Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"io task","client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166802024-01-24T19:12:49.499Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.352747758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166812024-01-24T19:12:49.499Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.352787511Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166822024-01-24T19:12:49.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.352879471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166832024-01-24T19:12:49.790Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.352910317Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166842024-01-24T19:12:49.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.353043117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166852024-01-24T19:12:49.790Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.353085777Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166862024-01-24T19:12:49.790Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.353734172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166872024-01-24T19:12:49.790Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.353946443Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166882024-01-24T19:12:49.790Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.354166382Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166892024-01-24T19:12:49.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.354507105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166902024-01-24T19:12:49.790Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.354546952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166912024-01-24T19:12:49.790Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:39.354570786Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166922024-01-24T19:12:49.790Z{Wait for a query_work_queue command to finish before sending IO
166932024-01-24T19:12:49.790Z"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.330710557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166942024-01-24T19:12:49.791Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.330773957Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166952024-01-24T19:12:49.791Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.33079731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166962024-01-24T19:12:49.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.330820255Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166972024-01-24T19:12:49.793Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.3308987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166982024-01-24T19:12:49.794Z{"msg":"[0]R flush_numbers[0..12]: [261, 266, 263, 261, 263, 267, 252, 263, 259, 265, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.330980364Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
166992024-01-24T19:12:49.794Z{"msg":"[0]R generation[0..12]: [150, 154, 152, 150, 152, 155, 145, 152, 149, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331032278Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167002024-01-24T19:12:49.794Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331079684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167012024-01-24T19:12:49.794Z{"msg":"[1]R flush_numbers[0..12]: [261, 266, 263, 261, 263, 267, 252, 263, 259, 265, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331143244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167022024-01-24T19:12:49.794Z{"msg":"[1]R generation[0..12]: [150, 154, 152, 150, 152, 155, 145, 152, 149, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331217573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167032024-01-24T19:12:49.794Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331284005Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167042024-01-24T19:12:49.794Z{"msg":"[2]R flush_numbers[0..12]: [261, 266, 263, 261, 263, 267, 252, 263, 259, 265, 264, 264]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331361873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167052024-01-24T19:12:49.795Z{"msg":"[2]R generation[0..12]: [150, 154, 152, 150, 152, 155, 145, 152, 149, 154, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331435976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167062024-01-24T19:12:49.796Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331508898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167072024-01-24T19:12:49.796Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331582543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167082024-01-24T19:12:49.796Zpid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167092024-01-24T19:12:49.796Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331695965Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167102024-01-24T19:12:49.796Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331747105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167112024-01-24T19:12:49.796Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331792987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167122024-01-24T19:12:49.796Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331838492Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167132024-01-24T19:12:49.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331881463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167142024-01-24T19:12:49.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331926677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167152024-01-24T19:12:49.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.331972014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167162024-01-24T19:12:49.798Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.332016781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167172024-01-24T19:12:49.798Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.33205982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167182024-01-24T19:12:49.798Z{"msg":"2c9109e9-58b9-4cf5-affc-6c780067354b is now active with session: ec7adabf-1325-4e0e-a64e-429c9d1f0f9d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.332104906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167192024-01-24T19:12:49.798Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.332148043Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753,"session_id":"ec7adabf-1325-4e0e-a64e-429c9d1f0f9d"}
167202024-01-24T19:12:49.799Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:47.33219154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1753}
167212024-01-24T19:12:49.799ZLoading write count information from file "/tmp/test_repair_verify.data"
167222024-01-24T19:12:49.799ZRun Repair workload
167232024-01-24T19:12:49.799Z01/30 Write block 517 len 4096 data: 17
167242024-01-24T19:12:49.799Z02/30 Flush
167252024-01-24T19:12:49.799Z03/30 Write block 284 len 24576 data: 13 14 13 13 14 13
167262024-01-24T19:12:49.799Z04/30 Write block 71 len 4096 data: 6
167272024-01-24T19:12:49.799Z05/30 Read block 227 len 12288
167282024-01-24T19:12:49.799Z06/30 Write block 541 len 4096 data: 12
167292024-01-24T19:12:49.799Z07/30 Flush
167302024-01-24T19:12:49.799Z08/30 Write block 235 len 24576 data: 10 12 11 9 9 9
167312024-01-24T19:12:49.799Z09/30 Read block 575 len 20480
167322024-01-24T19:12:49.799Z10/30 Read block 581 len 20480
167332024-01-24T19:12:49.799Z11/30 Write block 144 len 12288 data: 11 11 13
167342024-01-24T19:12:49.800Z12/30 Read block 247 len 36864
167352024-01-24T19:12:49.800Z13/30 Write block 375 len 36864 data: 14 12 12 8 9 12 13 11 12
167362024-01-24T19:12:49.800Z14/30 Read block 313 len 20480
167372024-01-24T19:12:49.800Z15/30 Read block 28 len 4096
167382024-01-24T19:12:49.800Z16/30 Flush
167392024-01-24T19:12:49.800Z17/30 Write block 294 len 28672 data: 13 11 13 11 13 11 11
167402024-01-24T19:12:49.800ZError: Difference in extent metadata found!
167412024-01-24T19:12:49.800Z18/30 Write block 290 len 8192 data: 11 9
167422024-01-24T19:12:49.800Z19/30 Read block 180 len 20480
167432024-01-24T19:12:49.800Z20/30 Read block 262 len 36864
167442024-01-24T19:12:49.800Z21/30 Write block 439 len 16384 data: 11 10 7 6
167452024-01-24T19:12:49.800Z22/30 Write block 424 len 24576 data: 11 11 9 13 14 16
167462024-01-24T19:12:49.800Z23/30 Write block 547 len 8192 data: 15 12
167472024-01-24T19:12:49.800Z24/30 Read block 165 len 32768
167482024-01-24T19:12:49.801Z25/30 Read block 85 len 8192
167492024-01-24T19:12:49.801Z26/30 Read block 460 len 28672
167502024-01-24T19:12:49.801Z27/30 Write block 129 len 16384 data: 7 11 12 12
167512024-01-24T19:12:49.801Z28/30 Write block 505 len 40960 data: 12 10 11 11 11 14 15 15 15 14
167522024-01-24T19:12:49.801Z29/30 Write block 151 len 32768 data: 15 17 15 14 12 13 15 16
167532024-01-24T19:12:49.801Z30/30 Write block 226 len 8192 data: 9 9
167542024-01-24T19:12:49.801Z----------------------------------------------------------------
167552024-01-24T19:12:49.801Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29
167562024-01-24T19:12:49.802ZStates: Active Active Active
167572024-01-24T19:12:49.803ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167582024-01-24T19:12:49.803Z 3 Acked 1002 Write 6 Done Done Done false
167592024-01-24T19:12:49.803Z 4 Acked 1003 Write 1 Done Done Done false
167602024-01-24T19:12:49.803Z 5 Acked 1004 Read 3 Done Sent Done false
167612024-01-24T19:12:49.803Z 6 Acked 1005 Write 1 Done Sent Done false
167622024-01-24T19:12:49.804Z 7 Acked 1006 Flush 0 Done Sent Done false
167632024-01-24T19:12:49.804Z 8 Acked 1007 Write 6 Done Sent Done false
167642024-01-24T19:12:49.804Z 9 Acked 1008 Read 5 Done Sent Done false
167652024-01-24T19:12:49.804Z 10 Acked 1009 Read 5 Done Sent Done false
167662024-01-24T19:12:49.804Z 11 Acked 1010 Write 3 Done Sent Done false
167672024-01-24T19:12:49.804Z 12 Acked 1011 Read 9 Done Sent Done false
167682024-01-24T19:12:49.804Z 13 Acked 1012 Write 9 Done Sent Done false
167692024-01-24T19:12:49.804Z 14 Acked 1013 Read 5 Done Sent Done false
167702024-01-24T19:12:49.804Z 15 Acked 1014 Read 1 Done Sent Done false
167712024-01-24T19:12:49.805Z 16 Acked 1015 Flush 0 Done Sent Done false
167722024-01-24T19:12:49.805Z 17 Acked 1016 Write 7 Done Sent Done false
167732024-01-24T19:12:49.805Z 18 Acked 1017 Write 2 Done Sent Done false
167742024-01-24T19:12:49.805Z 19 Acked 1018 Read 5 Done Sent Done false
167752024-01-24T19:12:49.805Z 20 Acked 1019 Read 9 Done Sent Done false
167762024-01-24T19:12:49.805Z 21 Acked 1020 Write 4 Done Sent Done false
167772024-01-24T19:12:49.805Z 22 Acked 1021 Flush 0 Done Sent Done false
167782024-01-24T19:12:49.805Z 23 Acked 1022 Write 6 Done Sent Done false
167792024-01-24T19:12:49.806Z 24 Acked 1023 Write 2 Done Sent Done false
167802024-01-24T19:12:49.806Z 25 Acked 1024 Read 8 Done Sent Done false
167812024-01-24T19:12:49.806Z 26 Acked 1025 Read 2 Done Sent Done false
167822024-01-24T19:12:49.931Z 27 Acked 1026 Read 7 Done Sent Done false
167832024-01-24T19:12:49.931Z 28 Acked 1027 Write 4 Done Sent Done false
167842024-01-24T19:12:49.931Z 29 Acked 1028 Write 10 Done Sent Done false
167852024-01-24T19:12:49.931Z 30 Acked 1029 Write 8 Sent Sent Sent false
167862024-01-24T19:12:49.931Z 31 Acked 1030 Write 2 Sent Sent Sent false
167872024-01-24T19:12:49.931Z STATES DS:0 DS:1 DS:2 TOTAL
167882024-01-24T19:12:49.931Z New 0 0 0 0
167892024-01-24T19:12:49.932Z Sent 2 27 2 31
167902024-01-24T19:12:49.932Z Done 27 2 27 56
167912024-01-24T19:12:49.932Z Skipped 0 0 0 0
167922024-01-24T19:12:49.932Z Error 0 0 0 0
167932024-01-24T19:12:49.932ZLast Flush: 1021 1001 1021
167942024-01-24T19:12:49.932ZDownstairs last five completed: 1001 1000
167952024-01-24T19:12:49.932ZUpstairs last five completed: 31 30 29 28 27
167962024-01-24T19:12:49.932ZWrote out file "/tmp/test_repair_verify.data"
167972024-01-24T19:12:49.933Z
167982024-01-24T19:12:49.933ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167992024-01-24T19:12:49.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168002024-01-24T19:12:49.933Z 0 000-019 150 150 150 261 261 261 F F F
168012024-01-24T19:12:49.933Z 1 020-039 154 154 154 266 266 266 F F F
168022024-01-24T19:12:49.933Z 2 040-059 152 152 152 263 263 263 F F F
168032024-01-24T19:12:49.934Z 3 060-079 156 150 156 269 261 269 F T F <---
168042024-01-24T19:12:49.934Z 4 080-099 152 152 152 263 263 263 F F F
168052024-01-24T19:12:49.934Z 5 100-119 155 155 155 267 267 267 F F F
168062024-01-24T19:12:49.934Z 6 120-139 145 145 145 252 252 252 T F T <---
168072024-01-24T19:12:49.934Z 7 140-159 156 152 156 270 263 270 T F T <---
168082024-01-24T19:12:49.934Z 8 160-179 149 149 149 259 259 259 F F F
168092024-01-24T19:12:49.934Z 9 180-199 154 154 154 265 265 265 F F F
168102024-01-24T19:12:49.934Z 10 200-219 153 153 153 264 264 264 F F F
168112024-01-24T19:12:49.934Z 11 220-239 156 153 156 270 264 270 T F T <---
168122024-01-24T19:12:49.934Z 12 240-259 156 154 156 270 266 270 F F F <---
168132024-01-24T19:12:49.934Z 13 260-279 150 150 150 261 261 261 F F F
168142024-01-24T19:12:49.935Z 14 280-299 156 154 156 271 266 271 F T F <---
168152024-01-24T19:12:49.935Z 15 300-319 156 154 156 271 266 271 F F F <---
168162024-01-24T19:12:49.935Z 16 320-339 153 153 153 264 264 264 F F F
168172024-01-24T19:12:49.935Z 17 340-359 153 153 153 264 264 264 F F F
168182024-01-24T19:12:49.935Z 18 360-379 156 154 156 270 265 270 F F F <---
168192024-01-24T19:12:49.935Z 19 380-399 156 154 156 270 265 270 F F F <---
168202024-01-24T19:12:49.935Z 20 400-419 153 153 153 264 264 264 F F F
168212024-01-24T19:12:49.935Z 21 420-439 156 154 156 271 265 271 T F T <---
168222024-01-24T19:12:49.935Z 22 440-459 156 155 156 271 267 271 F F F <---
168232024-01-24T19:12:49.936Z 23 460-479 155 155 155 267 267 267 F F F
168242024-01-24T19:12:49.936Z 24 480-499 152 152 152 263 263 263 F F F
168252024-01-24T19:12:49.936Z 25 500-519 156 156 156 268 268 268 T F T <---
168262024-01-24T19:12:49.936Z 26 520-539 154 154 154 266 266 266 F F F
168272024-01-24T19:12:49.936Z 27 540-559 156 154 156 269 266 269 T F T <---
168282024-01-24T19:12:49.936Z 28 560-579 153 153 153 264 264 264 F F F
168292024-01-24T19:12:49.936Z 29 580-599 149 149 149 259 259 259 F F F
168302024-01-24T19:12:49.936ZMax gen: 156, Max flush: 271
168312024-01-24T19:12:49.936ZA Difference in extent metadata is expected here
168322024-01-24T19:12:49.936ZOn loop 78
168332024-01-24T19:12:49.936Z
168342024-01-24T19:12:49.936ZVerifying data now
168352024-01-24T19:12:54.909ZNo change after verify
168362024-01-24T19:12:54.910ZLoop: 78 Downstairs dump after verify (and repair):
168372024-01-24T19:12:54.936ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168382024-01-24T19:12:54.936Z 0 000-019 150 150 150 261 261 261 F F F
168392024-01-24T19:12:54.936Z 1 020-039 154 154 154 266 266 266 F F F
168402024-01-24T19:12:54.936Z 2 040-059 152 152 152 263 263 263 F F F
168412024-01-24T19:12:54.936Z 3 060-079 156 156 156 269 269 269 F F F
168422024-01-24T19:12:54.936Z 4 080-099 152 152 152 263 263 263 F F F
168432024-01-24T19:12:54.936Z 5 100-119 155 155 155 267 267 267 F F F
168442024-01-24T19:12:54.937Z 6 120-139 157 157 157 272 272 272 F F F
168452024-01-24T19:12:54.937Z 7 140-159 157 157 157 272 272 272 F F F
168462024-01-24T19:12:54.937Z 8 160-179 149 149 149 259 259 259 F F F
168472024-01-24T19:12:54.937Z 9 180-199 154 154 154 265 265 265 F F F
168482024-01-24T19:12:54.937Z 10 200-219 153 153 153 264 264 264 F F F
168492024-01-24T19:12:54.937Z 11 220-239 157 157 157 272 272 272 F F F
168502024-01-24T19:12:54.937Z 12 240-259 156 156 156 270 270 270 F F F
168512024-01-24T19:12:54.937Z 13 260-279 150 150 150 261 261 261 F F F
168522024-01-24T19:12:54.937Z 14 280-299 156 156 156 271 271 271 F F F
168532024-01-24T19:12:54.937Z 15 300-319 156 156 156 271 271 271 F F F
168542024-01-24T19:12:54.937Z 16 320-339 153 153 153 264 264 264 F F F
168552024-01-24T19:12:54.937Z 17 340-359 153 153 153 264 264 264 F F F
168562024-01-24T19:12:54.937Z 18 360-379 156 156 156 270 270 270 F F F
168572024-01-24T19:12:54.937Z 19 380-399 156 156 156 270 270 270 F F F
168582024-01-24T19:12:54.937Z 20 400-419 153 153 153 264 264 264 F F F
168592024-01-24T19:12:54.937Z 21 420-439 157 157 157 272 272 272 F F F
168602024-01-24T19:12:54.937Z 22 440-459 156 156 156 271 271 271 F F F
168612024-01-24T19:12:54.937Z 23 460-479 155 155 155 267 267 267 F F F
168622024-01-24T19:12:54.937Z 24 480-499 152 152 152 263 263 263 F F F
168632024-01-24T19:12:54.938Z 25 500-519 157 157 157 272 272 272 F F F
168642024-01-24T19:12:54.938Z 26 520-539 154 154 154 266 266 266 F F F
168652024-01-24T19:12:54.938Z 27 540-559 157 157 157 272 272 272 F F F
168662024-01-24T19:12:54.938Z 28 560-579 153 153 153 264 264 264 F F F
168672024-01-24T19:12:54.938Z 29 580-599 149 149 149 259 259 259 F F F
168682024-01-24T19:12:54.938ZMax gen: 157, Max flush: 272
168692024-01-24T19:12:54.944Z
168702024-01-24T19:12:54.945ZBegin loop 79
168712024-01-24T19:12:54.945ZDownstairs to restart: 2
168722024-01-24T19:12:54.967Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.642718257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761}
168732024-01-24T19:12:54.967Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643189422Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761}
168742024-01-24T19:12:54.981Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.6432767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761}
168752024-01-24T19:12:54.981Z{"msg":"Crucible 2ed0f581-4fe8-4423-9264-54770855dba3 has session id: d7a012af-132f-4e0e-b063-e8c19b0d1e7b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643428389Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168762024-01-24T19:12:54.981Z{"msg":"Crucible runtime is spawned
168772024-01-24T19:12:54.981ZUpstairs opts: Upstairs UUID: 2ed0f581-4fe8-4423-9264-54770855dba3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643499453Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168782024-01-24T19:12:54.981Z{"msg":"Crucible stats registered with UUID: 2ed0f581-4fe8-4423-9264-54770855dba3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643544513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168792024-01-24T19:12:54.982Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643710003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761}
168802024-01-24T19:12:54.982Z{"msg":"2ed0f581-4fe8-4423-9264-54770855dba3 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643924201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168812024-01-24T19:12:54.982Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.643994331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168822024-01-24T19:12:54.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644040986Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168832024-01-24T19:12:54.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644117329Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168842024-01-24T19:12:54.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644163212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168852024-01-24T19:12:54.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644245735Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168862024-01-24T19:12:54.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644291082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168872024-01-24T19:12:54.983Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644330041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168882024-01-24T19:12:54.983Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644410536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168892024-01-24T19:12:54.985Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.64445871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168902024-01-24T19:12:54.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.64465603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168912024-01-24T19:12:54.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644759559Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168922024-01-24T19:12:54.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.644790996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"io task","client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168932024-01-24T19:12:54.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.646286731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168942024-01-24T19:12:54.985Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.646367933Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168952024-01-24T19:12:54.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.64654642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168962024-01-24T19:12:54.985Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.646641049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168972024-01-24T19:12:54.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.646781964Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168982024-01-24T19:12:54.985Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.646827414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
168992024-01-24T19:12:54.985Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.647383855Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169002024-01-24T19:12:54.985Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.647573082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169012024-01-24T19:12:54.985Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.647984834Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169022024-01-24T19:12:54.985Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648326622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169032024-01-24T19:12:54.985Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648409916Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169042024-01-24T19:12:54.985Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648453141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169052024-01-24T19:12:54.985Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648510903Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169062024-01-24T19:12:54.985Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648537461Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169072024-01-24T19:12:54.986Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648581738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169082024-01-24T19:12:54.986Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.648940139Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169092024-01-24T19:12:54.987Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649042572Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169102024-01-24T19:12:54.987Z{"msg":"[0]R flush_numbers[0..12]: [261, 266, 263, 269, 263, 267, 272, 272, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649088967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169112024-01-24T19:12:54.987Z{"msg":"[0]R generation[0..12]: [150, 154, 152, 156, 152, 155, 157, 157, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649163295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169122024-01-24T19:12:54.987Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649237999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169132024-01-24T19:12:54.987Z{"msg":"[1]R flush_numbers[0..12]: [261, 266, 263, 269, 263, 267, 272, 272, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649314845Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169142024-01-24T19:12:54.987Z{"msg"Wait for a query_work_queue command to finish before sending IO
169152024-01-24T19:12:54.987Z:"[1]R generation[0..12]: [150, 154, 152, 156, 152, 155, 157, 157, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649399506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169162024-01-24T19:12:54.987Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649478859Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169172024-01-24T19:12:54.987Z{"msg":"[2]R flush_numbers[0..12]: [261, 266, 263, 269, 263, 267, 272, 272, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649553178Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169182024-01-24T19:12:54.988Z{"msg":"[2]R generation[0..12]: [150, 154, 152, 156, 152, 155, 157, 157, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.64963287Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169192024-01-24T19:12:54.989Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649714281Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169202024-01-24T19:12:54.989Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649792547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169212024-01-24T19:12:54.989Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649866265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169222024-01-24T19:12:54.989Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.649939365Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169232024-01-24T19:12:54.989Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650011209Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169242024-01-24T19:12:54.989Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169252024-01-24T19:12:54.989Z,"time":"2024-01-24T19:12:53.650090729Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169262024-01-24T19:12:54.989Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650162632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169272024-01-24T19:12:54.989Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650191981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169282024-01-24T19:12:54.989Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650236864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169292024-01-24T19:12:54.989Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650262322Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169302024-01-24T19:12:54.992Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650303866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169312024-01-24T19:12:54.992Z{"msg":"2ed0f581-4fe8-4423-9264-54770855dba3 is now active with session: d7a012af-132f-4e0e-b063-e8c19b0d1e7b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650328558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169322024-01-24T19:12:54.993Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650369005Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761,"session_id":"d7a012af-132f-4e0e-b063-e8c19b0d1e7b"}
169332024-01-24T19:12:54.993Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:53.650392307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1761}
169342024-01-24T19:12:54.993ZLoading write count information from file "/tmp/test_repair_verify.data"
169352024-01-24T19:12:54.993ZRun Repair workload
169362024-01-24T19:12:54.993Z01/30 Write block 416 len 24576 data: 7 9 11 14 10 10
169372024-01-24T19:12:54.993Z02/30 Read block 155 len 40960
169382024-01-24T19:12:55.003Z03/30 Write block 537 len 4096 data: 12
169392024-01-24T19:12:55.003Z04/30 Write block 461 len 16384 data: 10 11 9 8
169402024-01-24T19:12:55.003Z05/30 Flush
169412024-01-24T19:12:55.003Z06/30 Flush
169422024-01-24T19:12:55.003Z07/30 Write block 134 len 32768 data: 13 13 14 14 13 13 10 9
169432024-01-24T19:12:55.003Z08/30 Read block 485 len 8192
169442024-01-24T19:12:55.003Z09/30 Flush
169452024-01-24T19:12:55.003Z10/30 Write block 514 len 28672 data: 15 14 12 18 17 16 15
169462024-01-24T19:12:55.003Z11/30 Write block 44 len 24576 data: 13 16 11 11 13 14
169472024-01-24T19:12:55.003Z12/30 Read block 268 len 32768
169482024-01-24T19:12:55.003Z13/30 Read block 448 len 40960
169492024-01-24T19:12:55.003Z14/30 Read block 191 len 40960
169502024-01-24T19:12:55.003Z15/30 Write block 515 len 8192 data: 15 13
169512024-01-24T19:12:55.010Z16/30 Write block 21 len 8192 data: 11 12
169522024-01-24T19:12:55.010Z17/30 Read block 449 len 20480
169532024-01-24T19:12:55.010Z18/30 Read block 187 len 20480
169542024-01-24T19:12:55.010Z19/30 Flush
169552024-01-24T19:12:55.010Z20/30 Flush
169562024-01-24T19:12:55.016Z21/30 Write block 121 len 28672 data: 11 11 10 14 12 10 8
169572024-01-24T19:12:55.016Z22/30 Flush
169582024-01-24T19:12:55.016Z23/30 Write block 39 len 8192 data: 7 7
169592024-01-24T19:12:55.016Z24/30 Read block 481 len 8192
169602024-01-24T19:12:55.016Z25/30 Read block 27 len 8192
169612024-01-24T19:12:55.025Z26/30 Write block 146 len 28672 data: 14 16 16 16 14 16 18
169622024-01-24T19:12:55.026Z27/30 Write block 440 len 8192 data: 11 8
169632024-01-24T19:12:55.026Z28/30 Write block 308 len 32768 data: 6 7 9 9 9 12 14 15
169642024-01-24T19:12:55.026Z29/30 Write block 419 len 36864 data: 15 11 11 12 12 12 12 10 14
169652024-01-24T19:12:55.041Z30/30 Write block 454 len 12288 data: 9 8 11
169662024-01-24T19:12:55.041Z----------------------------------------------------------------
169672024-01-24T19:12:55.041Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
169682024-01-24T19:12:55.041ZStates: Active Active Active
169692024-01-24T19:12:55.041ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169702024-01-24T19:12:55.041Z 1 Acked 1000 Write 6 Done Done Sent false
169712024-01-24T19:12:55.041Z 2 Acked 1001 Read 10 Done Done Sent false
169722024-01-24T19:12:55.041Z 3 Acked 1002 Write 1 Done Done Sent false
169732024-01-24T19:12:55.041Z 4 Acked 1003 Write 4 Done Done Sent false
169742024-01-24T19:12:55.041Z 5 Acked 1004 Flush 0 Done Done Sent false
169752024-01-24T19:12:55.041Z 6 Acked 1005 Flush 0 Done Done Sent false
169762024-01-24T19:12:55.041Z 7 Acked 1006 Write 8 Done Done Sent false
169772024-01-24T19:12:55.041Z 8 Acked 1007 Read 2 Done Done Sent false
169782024-01-24T19:12:55.041Z 9 Acked 1008 Flush 0 Done Done Sent false
169792024-01-24T19:12:55.041Z 10 Acked 1009 Write 7 Done Done Sent false
169802024-01-24T19:12:55.041Z 11 Acked 1010 Write 6 Done Done Sent false
169812024-01-24T19:12:55.041Z 12 Acked 1011 Read 8 Done Done Sent false
169822024-01-24T19:12:55.041Z 13 Acked 1012 Read 10 Done Done Sent false
169832024-01-24T19:12:55.041Z 14 Acked 1013 Read 10 Done Done Sent false
169842024-01-24T19:12:55.041Z 15 Acked 1014 Write 2 Done Done Sent false
169852024-01-24T19:12:55.041Z 16 Acked 1015 Write 2 Done Done Sent false
169862024-01-24T19:12:55.042Z 17 Acked 1016 Read 5 Done Done Sent false
169872024-01-24T19:12:55.042Z 18 Acked 1017 Read 5 Done Done Sent false
169882024-01-24T19:12:55.042Z 19 Acked 1018 Flush 0 Done Done Sent false
169892024-01-24T19:12:55.042Z 20 Acked 1019 Flush 0 Done Done Sent false
169902024-01-24T19:12:55.042Z 21 Acked 1020 Write 7 Done Done Sent false
169912024-01-24T19:12:55.042Z 22 Acked 1021 Flush 0 Done Done Sent false
169922024-01-24T19:12:55.042Z 23 Acked 1022 Write 2 Done Done Sent false
169932024-01-24T19:12:55.042Z 24 Acked 1023 Read 2 Done Done Sent false
169942024-01-24T19:12:55.042Z 25 Acked 1024 Read 2 Done Done Sent false
169952024-01-24T19:12:55.042Z 26 Acked 1025 Write 7 Done Done Sent false
169962024-01-24T19:12:55.042Z 27 Acked 1026 Write 2 Done Done Sent false
169972024-01-24T19:12:55.042Z 28 Acked 1027 Write 8 Done Done Sent false
169982024-01-24T19:12:55.042Z 29 Acked 1028 Write 9 Sent Sent Sent false
169992024-01-24T19:12:55.042Z 30 Acked 1029 Write 3 Sent Sent Sent false
170002024-01-24T19:12:55.042Z STATES DS:0 DS:1 DS:2 TOTAL
170012024-01-24T19:12:55.042Z New 0 0 0 0
170022024-01-24T19:12:55.042Z Sent 2 2 30 34
170032024-01-24T19:12:55.042Z Done 28 28 0 56
170042024-01-24T19:12:55.043Z Skipped 0 0 0 0
170052024-01-24T19:12:55.043Z Error 0 0 0 0
170062024-01-24T19:12:55.043ZLast Flush: 1021 1021 0
170072024-01-24T19:12:55.043ZDownstairs last five completed:
170082024-01-24T19:12:55.043ZUpstairs last five completed: 30 29 28 27 26
170092024-01-24T19:12:55.043ZWrote out file "/tmp/test_repair_verify.data"
170102024-01-24T19:12:55.043Z
170112024-01-24T19:12:55.043ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170122024-01-24T19:12:55.061ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170132024-01-24T19:12:55.061Z 0 000-019 150 150 150 261 261 261 F F F
170142024-01-24T19:12:55.079Z 1 020-039 158 158 154 276 276 266 T T F <---
170152024-01-24T19:12:55.080Z 2 040-059 158 158 152 276 276 263 T T F <---
170162024-01-24T19:12:55.080Z 3 060-079 156 156 156 269 269 269 F F F
170172024-01-24T19:12:55.080Z 4 080-099 152 152 152 263 263 263 F F F
170182024-01-24T19:12:55.080Z 5 100-119 155 155 155 267 267 267 F F F
170192024-01-24T19:12:55.080Z 6 120-139 158 158 157 278 278 272 F F F <---
170202024-01-24T19:12:55.080Z 7 140-159 158 158 157 275 275 272 T T F <---
170212024-01-24T19:12:55.080Z 8 160-179 149 149 149 259 259 259 F F F
170222024-01-24T19:12:55.082Z 9 180-199 154 154 154 265 265 265 F F F
170232024-01-24T19:12:55.082Z 10 200-219 153 153 153 264 264 264 F F F
170242024-01-24T19:12:55.082Z 11 220-239 157 157 157 272 272 272 F F F
170252024-01-24T19:12:55.082Z 12 240-259 156 156 156 270 270 270 F F F
170262024-01-24T19:12:55.082Z 13 260-279 150 150 150 261 261 261 F F F
170272024-01-24T19:12:55.082Z 14 280-299 156 156 156 271 271 271 F F F
170282024-01-24T19:12:55.082Z 15 300-319 156 156 156 271 271 271 T T F <---
170292024-01-24T19:12:55.082Z 16 320-339 153 153 153 264 264 264 F F F
170302024-01-24T19:12:55.082Z 17 340-359 153 153 153 264 264 264 F F F
170312024-01-24T19:12:55.082Z 18 360-379 156 156 156 270 270 270 F F F
170322024-01-24T19:12:55.083Z 19 380-399 156 156 156 270 270 270 F F F
170332024-01-24T19:12:55.083Z 20 400-419 158 158 153 273 273 264 T T F <---
170342024-01-24T19:12:55.083Z 21 420-439 158 158 157 273 273 272 T T F <---
170352024-01-24T19:12:55.083Z 22 440-459 156 156 156 271 271 271 T T F <---
170362024-01-24T19:12:55.083Z 23 460-479 158 158 155 273 273 267 F F F <---
170372024-01-24T19:12:55.083Z 24 480-499 152 152 152 263 263 263 F F F
170382024-01-24T19:12:55.083Z 25 500-519 158 158 157 276 276 272 F F F <---
170392024-01-24T19:12:55.083Z 26 520-539 158 158 154 276 276 266 F F F <---
170402024-01-24T19:12:55.083Z 27 540-559 157 157 157 272 272 272 F F F
170412024-01-24T19:12:55.083Z 28 560-579 153 153 153 264 264 264 F F F
170422024-01-24T19:12:55.083Z 29 580-599 149 149 149 259 259 259 F F F
170432024-01-24T19:12:55.083ZMax gen: 158, Max flush: 278
170442024-01-24T19:12:55.083ZError: Difference in extent metadata found!
170452024-01-24T19:12:55.083ZA Difference in extent metadata is expected here
170462024-01-24T19:12:55.086ZOn loop 79
170472024-01-24T19:12:55.086Z
170482024-01-24T19:12:55.087ZVerifying data now
170492024-01-24T19:13:00.071ZNo change after verify
170502024-01-24T19:13:00.071ZLoop: 79 Downstairs dump after verify (and repair):
170512024-01-24T19:13:00.097ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170522024-01-24T19:13:00.097Z 0 000-019 150 150 150 261 261 261 F F F
170532024-01-24T19:13:00.097Z 1 020-039 159 159 159 279 279 279 F F F
170542024-01-24T19:13:00.098Z 2 040-059 159 159 159 279 279 279 F F F
170552024-01-24T19:13:00.098Z 3 060-079 156 156 156 269 269 269 F F F
170562024-01-24T19:13:00.098Z 4 080-099 152 152 152 263 263 263 F F F
170572024-01-24T19:13:00.103Z 5 100-119 155 155 155 267 267 267 F F F
170582024-01-24T19:13:00.103Z 6 120-139 158 158 158 278 278 278 F F F
170592024-01-24T19:13:00.103Z 7 140-159 159 159 159 279 279 279 F F F
170602024-01-24T19:13:00.103Z 8 160-179 149 149 149 259 259 259 F F F
170612024-01-24T19:13:00.104Z 9 180-199 154 154 154 265 265 265 F F F
170622024-01-24T19:13:00.104Z 10 200-219 153 153 153 264 264 264 F F F
170632024-01-24T19:13:00.104Z 11 220-239 157 157 157 272 272 272 F F F
170642024-01-24T19:13:00.104Z 12 240-259 156 156 156 270 270 270 F F F
170652024-01-24T19:13:00.105Z 13 260-279 150 150 150 261 261 261 F F F
170662024-01-24T19:13:00.105Z 14 280-299 156 156 156 271 271 271 F F F
170672024-01-24T19:13:00.105Z 15 300-319 159 159 159 279 279 279 F F F
170682024-01-24T19:13:00.105Z 16 320-339 153 153 153 264 264 264 F F F
170692024-01-24T19:13:00.105Z 17 340-359 153 153 153 264 264 264 F F F
170702024-01-24T19:13:00.106Z 18 360-379 156 156 156 270 270 270 F F F
170712024-01-24T19:13:00.106Z 19 380-399 156 156 156 270 270 270 F F F
170722024-01-24T19:13:00.106Z 20 400-419 159 159 159 279 279 279 F F F
170732024-01-24T19:13:00.106Z 21 420-439 159 159 159 279 279 279 F F F
170742024-01-24T19:13:00.106Z 22 440-459 159 159 159 279 279 279 F F F
170752024-01-24T19:13:00.107Z 23 460-479 158 158 158 273 273 273 F F F
170762024-01-24T19:13:00.107Z 24 480-499 152 152 152 263 263 263 F F F
170772024-01-24T19:13:00.107Z 25 500-519 158 158 158 276 276 276 F F F
170782024-01-24T19:13:00.107Z 26 520-539 158 158 158 276 276 276 F F F
170792024-01-24T19:13:00.107Z 27 540-559 157 157 157 272 272 272 F F F
170802024-01-24T19:13:00.108Z 28 560-579 153 153 153 264 264 264 F F F
170812024-01-24T19:13:00.108Z 29 580-599 149 149 149 259 259 259 F F F
170822024-01-24T19:13:00.108ZMax gen: 159, Max flush: 279
170832024-01-24T19:13:00.108Z
170842024-01-24T19:13:00.109ZBegin loop 80
170852024-01-24T19:13:00.109ZDownstairs to restart: 0
170862024-01-24T19:13:00.140Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.804667469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769}
170872024-01-24T19:13:00.148Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805152605Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769}
170882024-01-24T19:13:00.154Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805233681Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769}
170892024-01-24T19:13:00.154Z{"msg":"Crucible 787f0d5d-8616-4df2-a72f-7b1951251251 has session id: cb82fdb0-09cb-4689-88bd-a4e1b70a35a3","v":0,"name":"crucible","level":30Crucible runtime is spawned
170902024-01-24T19:13:00.154Z,"time":"2024-01-24T19:12:58.805326843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170912024-01-24T19:13:00.154Z{"msg":"Upstairs opts: Upstairs UUID: 787f0d5d-8616-4df2-a72f-7b1951251251, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805481061Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170922024-01-24T19:13:00.154Z{"msg":"Crucible stats registered with UUID: 787f0d5d-8616-4df2-a72f-7b1951251251","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805545449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170932024-01-24T19:13:00.155Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805576978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769}
170942024-01-24T19:13:00.155Z{"msg":"787f0d5d-8616-4df2-a72f-7b1951251251 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805720243Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170952024-01-24T19:13:00.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805780983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170962024-01-24T19:13:00.155Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805808946Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170972024-01-24T19:13:00.155Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805899269Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170982024-01-24T19:13:00.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.805940011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
170992024-01-24T19:13:00.157Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806066337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171002024-01-24T19:13:00.157Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806110203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171012024-01-24T19:13:00.157Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806180767Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171022024-01-24T19:13:00.157Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.8064455Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171032024-01-24T19:13:00.157Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806494442Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171042024-01-24T19:13:00.157Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806562984Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171052024-01-24T19:13:00.158Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806609269Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171062024-01-24T19:13:00.158Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.806672409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171072024-01-24T19:13:00.158Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.807552758Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171082024-01-24T19:13:00.159Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.807620689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171092024-01-24T19:13:00.159Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.807753488Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171102024-01-24T19:13:00.159Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.80782962Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171112024-01-24T19:13:00.159Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.808099166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171122024-01-24T19:13:00.159Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.808223406Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171132024-01-24T19:13:00.159Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.808709237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171142024-01-24T19:13:00.159Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.808782486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171152024-01-24T19:13:00.159Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809249376Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171162024-01-24T19:13:00.159Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809646632Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171172024-01-24T19:13:00.160Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809713547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171182024-01-24T19:13:00.168Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809744876Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171192024-01-24T19:13:00.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809822719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171202024-01-24T19:13:00.168Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809895058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171212024-01-24T19:13:00.168Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.809971652Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171222024-01-24T19:13:00.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810314687Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171232024-01-24T19:13:00.168Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.81040467Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171242024-01-24T19:13:00.168Z{"msg":"[0]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 267, 278, 279, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810483582Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171252024-01-24T19:13:00.168Z{"msg":"[0]R generation[0..12]: [150, 159, 159, 156, 152, 155, 158, 159, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810558312Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171262024-01-24T19:13:00.168Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810631211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pidWait for a query_work_queue command to finish before sending IO
171272024-01-24T19:13:00.168Z":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171282024-01-24T19:13:00.168Z{"msg":"[1]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 267, 278, 279, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810716497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171292024-01-24T19:13:00.168Z{"msg":"[1]R generation[0..12]: [150, 159, 159, 156, 152, 155, 158, 159, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.81081466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171302024-01-24T19:13:00.168Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810863576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171312024-01-24T19:13:00.168Z{"msg":"[2]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 267, 278, 279, 259, 265, 264, 272]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.810942503Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171322024-01-24T19:13:00.168Z{"msg":"[2]R generation[0..12]: [150, 159, 159, 156, 152, 155, 158, 159, 149, 154, 153, 157]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811022229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171332024-01-24T19:13:00.168Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811104184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171342024-01-24T19:13:00.168Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811179095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171352024-01-24T19:13:00.168Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811250637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171362024-01-24T19:13:00.169Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171372024-01-24T19:13:00.169Z,"time":"2024-01-24T19:12:58.811329639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171382024-01-24T19:13:00.169Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811407727Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171392024-01-24T19:13:00.169Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811464624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171402024-01-24T19:13:00.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811489912Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171412024-01-24T19:13:00.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811533594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171422024-01-24T19:13:00.169Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811559069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171432024-01-24T19:13:00.169Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811601507Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171442024-01-24T19:13:00.169Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811625406Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171452024-01-24T19:13:00.169Z{"msg":"787f0d5d-8616-4df2-a72f-7b1951251251 is now active with session: cb82fdb0-09cb-4689-88bd-a4e1b70a35a3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811668213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171462024-01-24T19:13:00.169Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.81169172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769,"session_id":"cb82fdb0-09cb-4689-88bd-a4e1b70a35a3"}
171472024-01-24T19:13:00.169Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:12:58.811732862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1769}
171482024-01-24T19:13:00.169ZLoading write count information from file "/tmp/test_repair_verify.data"
171492024-01-24T19:13:00.169ZRun Repair workload
171502024-01-24T19:13:00.169Z01/30 Write block 119 len 24576 data: 11 10 12 12 11 15
171512024-01-24T19:13:00.169Z02/30 Read block 510 len 4096
171522024-01-24T19:13:00.169Z03/30 Read block 135 len 36864
171532024-01-24T19:13:00.169Z04/30 Read block 69 len 4096
171542024-01-24T19:13:00.169Z05/30 Read block 38 len 36864
171552024-01-24T19:13:00.175Z06/30 Read block 456 len 36864
171562024-01-24T19:13:00.175Z07/30 Read block 80 len 8192
171572024-01-24T19:13:00.175Z08/30 Read block 360 len 12288
171582024-01-24T19:13:00.175Z09/30 Read block 437 len 12288
171592024-01-24T19:13:00.175Z10/30 Read block 179 len 40960
171602024-01-24T19:13:00.175Z11/30 Write block 573 len 36864 data: 12 11 12 12 7 9 10 10 10
171612024-01-24T19:13:00.175Z12/30 Write block 234 len 24576 data: 14 11 13 12 10 10
171622024-01-24T19:13:00.175Z13/30 Write block 130 len 28672 data: 12 13 13 10 14 14 15
171632024-01-24T19:13:00.176Z14/30 Read block 467 len 4096
171642024-01-24T19:13:00.176Z15/30 Read block 129 len 20480
171652024-01-24T19:13:00.176Z16/30 Write block 397 len 24576 data: 12 11 10 11 12 11
171662024-01-24T19:13:00.176Z17/30 Write block 399 len 24576 data: 11 12 13 12 7 6
171672024-01-24T19:13:00.176Z18/30 Flush
171682024-01-24T19:13:00.176Z19/30 Write block 267 len 8192 data: 12 11
171692024-01-24T19:13:00.176Z20/30 Read block 570 len 36864
171702024-01-24T19:13:00.176Z21/30 Read block 139 len 40960
171712024-01-24T19:13:00.176Z22/30 Write block 537 len 12288 data: 13 12 13
171722024-01-24T19:13:00.176Z23/30 Write block 588 len 36864 data: 11 9 9 11 14 13 12 9 11
171732024-01-24T19:13:00.180Z24/30 Write block 555 len 32768 data: 10 11 13 13 13 10 13 10
171742024-01-24T19:13:00.180Z25/30 Read block 579 len 8192
171752024-01-24T19:13:00.191Z26/30 Read block 190 len 24576
171762024-01-24T19:13:00.191Z27/30 Read block 292 len 24576
171772024-01-24T19:13:00.196Z28/30 Write block 322 len 16384 data: 11 12 11 13
171782024-01-24T19:13:00.196Z29/30 Write block 522 len 28672 data: 12 11 13 14 14 12 17
171792024-01-24T19:13:00.196Z30/30 Write block 569 len 40960 data: 19 13 13 15 13 12 13 13 8 10
171802024-01-24T19:13:00.196Z----------------------------------------------------------------
171812024-01-24T19:13:00.196Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
171822024-01-24T19:13:00.196ZStates: Active Active Active
171832024-01-24T19:13:00.196ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
171842024-01-24T19:13:00.196Z 1 Acked 1000 Write 6 Sent Done Done false
171852024-01-24T19:13:00.196Z 2 Acked 1001 Read 1 Sent Done Done false
171862024-01-24T19:13:00.196Z 3 Acked 1002 Read 9 Sent Done Done false
171872024-01-24T19:13:00.196Z 4 Acked 1003 Read 1 Sent Done Done false
171882024-01-24T19:13:00.196Z 5 Acked 1004 Read 9 Sent Done Done false
171892024-01-24T19:13:00.196Z 6 Acked 1005 Read 9 Sent Done Done false
171902024-01-24T19:13:00.196Z 7 Acked 1006 Read 2 Sent Done Done false
171912024-01-24T19:13:00.196Z 8 Acked 1007 Read 3 Sent Done Done false
171922024-01-24T19:13:00.196Z 9 Acked 1008 Read 3 Sent Done Done false
171932024-01-24T19:13:00.196Z 10 Acked 1009 Read 10 Sent Done Done false
171942024-01-24T19:13:00.197Z 11 Acked 1010 Write 9 Sent Done Done false
171952024-01-24T19:13:00.197Z 12 Acked 1011 Write 6 Sent Done Done false
171962024-01-24T19:13:00.197Z 13 Acked 1012 Write 7 Sent Done Done false
171972024-01-24T19:13:00.197Z 14 Acked 1013 Read 1 Sent Done Done false
171982024-01-24T19:13:00.197Z 15 Acked 1014 Read 5 Sent Done Done false
171992024-01-24T19:13:00.197Z 16 Acked 1015 Write 6 Sent Done Done false
172002024-01-24T19:13:00.197Z 17 Acked 1016 Write 6 Sent Done Done false
172012024-01-24T19:13:00.197Z 18 Acked 1017 Flush 0 Sent Done Done false
172022024-01-24T19:13:00.198Z 19 Acked 1018 Write 2 Sent Done Done false
172032024-01-24T19:13:00.198Z 20 Acked 1019 Read 9 Sent Done Done false
172042024-01-24T19:13:00.198Z 21 Acked 1020 Read 10 Sent Done Done false
172052024-01-24T19:13:00.198Z 22 Acked 1021 Write 3 Sent Done Done false
172062024-01-24T19:13:00.198Z 23 Acked 1022 Write 9 Sent Done Done false
172072024-01-24T19:13:00.198Z 24 Acked 1023 Write 8 Sent Done Done false
172082024-01-24T19:13:00.199Z 25 Acked 1024 Read 2 Sent Done Done false
172092024-01-24T19:13:00.199Z 26 Acked 1025 Read 6 Sent Done Done false
172102024-01-24T19:13:00.199Z 27 Acked 1026 Read 6 Sent Done Done false
172112024-01-24T19:13:00.199Z 28 Acked 1027 Write 4 Sent Done Done false
172122024-01-24T19:13:00.199Z 29 Acked 1028 Write 7 Sent Done Done false
172132024-01-24T19:13:00.199Z 30 Acked 1029 Write 10 Sent Sent Sent false
172142024-01-24T19:13:00.199Z STATES DS:0 DS:1 DS:2 TOTAL
172152024-01-24T19:13:00.199Z New 0 0 0 0
172162024-01-24T19:13:00.199Z Sent 30 1 1 32
172172024-01-24T19:13:00.199Z Done 0 29 29 58
172182024-01-24T19:13:00.199Z Skipped 0 0 0 0
172192024-01-24T19:13:00.199Z Error 0 0 0 0
172202024-01-24T19:13:00.199ZLast Flush: 0 1017 1017
172212024-01-24T19:13:00.199ZDownstairs last five completed:
172222024-01-24T19:13:00.199ZUpstairs last five completed: 30 29 28 27 26
172232024-01-24T19:13:00.199ZWrote out file "/tmp/test_repair_verify.data"
172242024-01-24T19:13:00.199Z
172252024-01-24T19:13:00.225ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172262024-01-24T19:13:00.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172272024-01-24T19:13:00.226Z 0 000-019 150 150 150 261 261 261 F F F
172282024-01-24T19:13:00.226Z 1 020-039 159 159 159 279 279 279 F F F
172292024-01-24T19:13:00.226Z 2 040-059 159 159 159 279 279 279 F F F
172302024-01-24T19:13:00.226Z 3 060-079 156 156 156 269 269 269 F F F
172312024-01-24T19:13:00.226Z 4 080-099 152 152 152 263 263 263 F F F
172322024-01-24T19:13:00.226Z 5 100-119 155 160 160 267 280 280 F F F <---
172332024-01-24T19:13:00.226Z 6 120-139 158 160 160 278 280 280 F F F <---
172342024-01-24T19:13:00.226Z 7 140-159 159 159 159 279 279 279 F F F
172352024-01-24T19:13:00.226Z 8 160-179 149 149 149 259 259 259 F F F
172362024-01-24T19:13:00.226Z 9 180-199 154 154 154 265 265 265 F F F
172372024-01-24T19:13:00.226Z 10 200-219 153 153 153 264 264 264 F F F
172382024-01-24T19:13:00.226Z 11 220-239 157 160 160 272 280 280 F F F <---
172392024-01-24T19:13:00.226Z 12 240-259 156 156 156 270 270 270 F F F
172402024-01-24T19:13:00.226Z 13 260-279 150 150 150 261 261 261 F T T <---
172412024-01-24T19:13:00.226Z 14 280-299 156 156 156 271 271 271 F F F
172422024-01-24T19:13:00.226Z 15 300-319 159 159 159 279 279 279 F F F
172432024-01-24T19:13:00.226Z 16 320-339 153 153 153 264 264 264 F T T <---
172442024-01-24T19:13:00.226Z 17 340-359 153 153 153 264 264 264 F F F
172452024-01-24T19:13:00.226Z 18 360-379 156 156 156 270 270 270 F F F
172462024-01-24T19:13:00.226Z 19 380-399 156 160 160 270 280 280 F F F <---
172472024-01-24T19:13:00.228Z 20 400-419 159 160 160 279 280 280 F F F <---
172482024-01-24T19:13:00.228Z 21 420-439 159 159 159 279 279 279 F F F
172492024-01-24T19:13:00.228Z 22 440-459 159 159 159 279 279 279 F F F
172502024-01-24T19:13:00.228Z 23 460-479 158 158 158 273 273 273 F F F
172512024-01-24T19:13:00.228Z 24 480-499 152 152 152 263 263 263 F F F
172522024-01-24T19:13:00.228Z 25 500-519 158 158 158 276 276 276 F F F
172532024-01-24T19:13:00.228Z 26 520-539 158 158 158 276 276 276 F T T <---
172542024-01-24T19:13:00.228Z 27 540-559 157 157 157 272 272 272 F T T <---
172552024-01-24T19:13:00.228Z 28 560-579 153 160 160 264 280 280 F T T <---
172562024-01-24T19:13:00.228Z 29 580-599 149 160 160 259 280 280 F T T <---
172572024-01-24T19:13:00.228ZMax gen: 160, Max flush: 280
172582024-01-24T19:13:00.234ZError: Difference in extent metadata found!
172592024-01-24T19:13:00.234ZA Difference in extent metadata is expected here
172602024-01-24T19:13:00.234ZOn loop 80
172612024-01-24T19:13:00.234Z
172622024-01-24T19:13:00.234ZVerifying data now
172632024-01-24T19:13:05.294ZNo change after verify
172642024-01-24T19:13:05.294ZLoop: 80 Downstairs dump after verify (and repair):
172652024-01-24T19:13:05.310ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172662024-01-24T19:13:05.310Z 0 000-019 150 150 150 261 261 261 F F F
172672024-01-24T19:13:05.310Z 1 020-039 159 159 159 279 279 279 F F F
172682024-01-24T19:13:05.310Z 2 040-059 159 159 159 279 279 279 F F F
172692024-01-24T19:13:05.310Z 3 060-079 156 156 156 269 269 269 F F F
172702024-01-24T19:13:05.310Z 4 080-099 152 152 152 263 263 263 F F F
172712024-01-24T19:13:05.310Z 5 100-119 160 160 160 280 280 280 F F F
172722024-01-24T19:13:05.310Z 6 120-139 160 160 160 280 280 280 F F F
172732024-01-24T19:13:05.310Z 7 140-159 159 159 159 279 279 279 F F F
172742024-01-24T19:13:05.310Z 8 160-179 149 149 149 259 259 259 F F F
172752024-01-24T19:13:05.310Z 9 180-199 154 154 154 265 265 265 F F F
172762024-01-24T19:13:05.310Z 10 200-219 153 153 153 264 264 264 F F F
172772024-01-24T19:13:05.311Z 11 220-239 160 160 160 280 280 280 F F F
172782024-01-24T19:13:05.311Z 12 240-259 156 156 156 270 270 270 F F F
172792024-01-24T19:13:05.311Z 13 260-279 161 161 161 281 281 281 F F F
172802024-01-24T19:13:05.311Z 14 280-299 156 156 156 271 271 271 F F F
172812024-01-24T19:13:05.311Z 15 300-319 159 159 159 279 279 279 F F F
172822024-01-24T19:13:05.311Z 16 320-339 161 161 161 281 281 281 F F F
172832024-01-24T19:13:05.312Z 17 340-359 153 153 153 264 264 264 F F F
172842024-01-24T19:13:05.312Z 18 360-379 156 156 156 270 270 270 F F F
172852024-01-24T19:13:05.312Z 19 380-399 160 160 160 280 280 280 F F F
172862024-01-24T19:13:05.312Z 20 400-419 160 160 160 280 280 280 F F F
172872024-01-24T19:13:05.312Z 21 420-439 159 159 159 279 279 279 F F F
172882024-01-24T19:13:05.312Z 22 440-459 159 159 159 279 279 279 F F F
172892024-01-24T19:13:05.312Z 23 460-479 158 158 158 273 273 273 F F F
172902024-01-24T19:13:05.312Z 24 480-499 152 152 152 263 263 263 F F F
172912024-01-24T19:13:05.312Z 25 500-519 158 158 158 276 276 276 F F F
172922024-01-24T19:13:05.312Z 26 520-539 161 161 161 281 281 281 F F F
172932024-01-24T19:13:05.312Z 27 540-559 161 161 161 281 281 281 F F F
172942024-01-24T19:13:05.312Z 28 560-579 161 161 161 281 281 281 F F F
172952024-01-24T19:13:05.312Z 29 580-599 161 161 161 281 281 281 F F F
172962024-01-24T19:13:05.312ZMax gen: 161, Max flush: 281
172972024-01-24T19:13:05.312Z
172982024-01-24T19:13:05.312ZBegin loop 81
172992024-01-24T19:13:05.312ZDownstairs to restart: 2
173002024-01-24T19:13:05.312Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.969398258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777}
173012024-01-24T19:13:05.312Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.969927684Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777}
173022024-01-24T19:13:05.312Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.969995191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777}
173032024-01-24T19:13:05.314Z{"msg":"Crucible dfbd3bc8-cb43-457c-b25d-f837a7e13f00 has session id: 520208a6-8e45-49ac-bc7d-59026f1af6f1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970036753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173042024-01-24T19:13:05.314Z{"Crucible runtime is spawned
173052024-01-24T19:13:05.314Zmsg":"Upstairs opts: Upstairs UUID: dfbd3bc8-cb43-457c-b25d-f837a7e13f00, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970219095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173062024-01-24T19:13:05.314Z{"msg":"Crucible stats registered with UUID: dfbd3bc8-cb43-457c-b25d-f837a7e13f00","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970296177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173072024-01-24T19:13:05.314Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.97033945Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777}
173082024-01-24T19:13:05.314Z{"msg":"dfbd3bc8-cb43-457c-b25d-f837a7e13f00 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970616265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173092024-01-24T19:13:05.314Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970682212Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173102024-01-24T19:13:05.314Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970766983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173112024-01-24T19:13:05.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970832299Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173122024-01-24T19:13:05.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970858705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173132024-01-24T19:13:05.315Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970890509Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173142024-01-24T19:13:05.315Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970914764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173152024-01-24T19:13:05.315Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.970936666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173162024-01-24T19:13:05.315Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.971024954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173172024-01-24T19:13:05.315Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.971073201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173182024-01-24T19:13:05.315Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.971220258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173192024-01-24T19:13:05.315Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.97135672Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173202024-01-24T19:13:05.315Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.971435922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173212024-01-24T19:13:05.315Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.972632612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173222024-01-24T19:13:05.315Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.972715213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173232024-01-24T19:13:05.318Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.972972314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173242024-01-24T19:13:05.318Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.973060259Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173252024-01-24T19:13:05.318Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.973420479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173262024-01-24T19:13:05.318Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.973532297Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173272024-01-24T19:13:05.319Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.973600211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173282024-01-24T19:13:05.319Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974008485Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173292024-01-24T19:13:05.319Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974417576Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173302024-01-24T19:13:05.319Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974720585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173312024-01-24T19:13:05.319Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974811658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173322024-01-24T19:13:05.319Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974893593Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173332024-01-24T19:13:05.321Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974934999Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173342024-01-24T19:13:05.321Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.974972056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173352024-01-24T19:13:05.321Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975039813Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173362024-01-24T19:13:05.321Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975239601Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173372024-01-24T19:13:05.321Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173382024-01-24T19:13:05.321Z{"msg":"[0]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 280, 280, 279, 259, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975377272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173392024-01-24T19:13:05.321Z{"msg":"[0]R generation[0..12]: [150, 159, 159, 156, 152, 160, 160, 159, 149, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975441948Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173402024-01-24T19:13:05.321Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975489014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173412024-01-24T19:13:05.321Z{"msg":"[1]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 280, 280, 279, 259, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975545318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173422024-01-24T19:13:05.321Z{"msg":"[1]R generation[0..12]: [150, 159, 159, 156, 152, 160, 160, 159, 149, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975589198Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
173432024-01-24T19:13:05.323Z520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173442024-01-24T19:13:05.323Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975690437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173452024-01-24T19:13:05.323Z{"msg":"[2]R flush_numbers[0..12]: [261, 279, 279, 269, 263, 280, 280, 279, 259, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.97575678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173462024-01-24T19:13:05.323Z{"msg":"[2]R generation[0..12]: [150, 159, 159, 156, 152, 160, 160, 159, 149, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975802981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173472024-01-24T19:13:05.323Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975861116Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173482024-01-24T19:13:05.323Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975903949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173492024-01-24T19:13:05.323Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.975961219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173502024-01-24T19:13:05.324Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976003008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173512024-01-24T19:13:05.324Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976096316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173522024-01-24T19:13:05.324Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976122196Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173532024-01-24T19:13:05.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976200904Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173542024-01-24T19:13:05.327Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976226937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173552024-01-24T19:13:05.328Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976248808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173562024-01-24T19:13:05.328Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976297437Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173572024-01-24T19:13:05.328Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976323282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173582024-01-24T19:13:05.328Z{"msg":"dfbd3bc8-cb43-457c-b25d-f837a7e13f00 is now active with session: 520208a6-8e45-49ac-bc7d-59026f1af6f1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976344902Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173592024-01-24T19:13:05.328Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976398234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777,"session_id":"520208a6-8e45-49ac-bc7d-59026f1af6f1"}
173602024-01-24T19:13:05.328Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:03.976445174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1777}
173612024-01-24T19:13:05.328ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
173622024-01-24T19:13:05.328ZLoading write count information from file "/tmp/test_repair_verify.data"
173632024-01-24T19:13:05.328ZRun Repair workload
173642024-01-24T19:13:05.328Z01/30 Write block 160 len 36864 data: 11 19 18 19 15 17 15 17 18
173652024-01-24T19:13:05.402Z02/30 Write block 566 len 8192 data: 12 17
173662024-01-24T19:13:05.403Z03/30 Read block 158 len 20480
173672024-01-24T19:13:05.403Z04/30 Write block 165 len 24576 data: 18 16 18 19 16 16
173682024-01-24T19:13:05.403Z05/30 Write block 78 len 36864 data: 11 11 11 13 11 14 15 12 13
173692024-01-24T19:13:05.404Z06/30 Read block 95 len 32768
173702024-01-24T19:13:05.404Z07/30 Write block 509 len 24576 data: 12 15 16 16 16 16
173712024-01-24T19:13:05.404Z08/30 Flush
173722024-01-24T19:13:05.404Z09/30 Flush
173732024-01-24T19:13:05.404Z10/30 Write block 243 len 8192 data: 15 15
173742024-01-24T19:13:05.404Z11/30 Read block 312 len 12288
173752024-01-24T19:13:05.404Z12/30 Write block 299 len 4096 data: 12
173762024-01-24T19:13:05.404Z13/30 Read block 571 len 40960
173772024-01-24T19:13:05.404Z14/30 Read block 483 len 8192
173782024-01-24T19:13:05.404Z15/30 Read block 141 len 40960
173792024-01-24T19:13:05.404Z16/30 Write block 165 len 12288 data: 19 17 19
173802024-01-24T19:13:05.404Z17/30 Write block 9 len 28672 data: 9 9 8 7 7 7 8
173812024-01-24T19:13:05.405Z18/30 Write block 334 len 28672 data: 11 12 12 12 12 12 12
173822024-01-24T19:13:05.405Z19/30 Read block 337 len 28672
173832024-01-24T19:13:05.405Z20/30 Write block 550 len 20480 data: 11 12 11 12 12
173842024-01-24T19:13:05.405Z21/30 Write block 534 len 12288 data: 11 10 10
173852024-01-24T19:13:05.405Z22/30 Read block 402 len 8192
173862024-01-24T19:13:05.405Z23/30 Flush
173872024-01-24T19:13:05.405Z24/30 Write block 39 len 36864 data: 8 8 11 10 11 14 17 12 12
173882024-01-24T19:13:05.405Z25/30 Write block 507 len 8192 data: 12 12
173892024-01-24T19:13:05.405Z26/30 Read block 526 len 36864
173902024-01-24T19:13:05.405Z27/30 Read block 123 len 8192
173912024-01-24T19:13:05.405Z28/30 Write block 451 len 12288 data: 8 9 10
173922024-01-24T19:13:05.405Z29/30 Write block 133 len 32768 data: 11 15 15 16 15 14 14 11
173932024-01-24T19:13:05.405Z30/30 Write block 411 len 16384 data: 12 11 10 8
173942024-01-24T19:13:05.406Z----------------------------------------------------------------
173952024-01-24T19:13:05.406Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
173962024-01-24T19:13:05.406ZStates: Active Active Active
173972024-01-24T19:13:05.406ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
173982024-01-24T19:13:05.406Z 1 Acked 1000 Write 9 Done Done Done false
173992024-01-24T19:13:05.407Z 2 Acked 1001 Write 2 Done Done Done false
174002024-01-24T19:13:05.407Z 3 Acked 1002 Read 5 Done Done Sent false
174012024-01-24T19:13:05.407Z 4 Acked 1003 Write 6 Done Done Sent false
174022024-01-24T19:13:05.407Z 5 Acked 1004 Write 9 Done Done Sent false
174032024-01-24T19:13:05.407Z 6 Acked 1005 Read 8 Done Done Sent false
174042024-01-24T19:13:05.407Z 7 Acked 1006 Write 6 Done Done Sent false
174052024-01-24T19:13:05.407Z 8 Acked 1007 Flush 0 Done Done Sent false
174062024-01-24T19:13:05.407Z 9 Acked 1008 Flush 0 Done Done Sent false
174072024-01-24T19:13:05.407Z 10 Acked 1009 Write 2 Done Done Sent false
174082024-01-24T19:13:05.407Z 11 Acked 1010 Read 3 Done Done Sent false
174092024-01-24T19:13:05.407Z 12 Acked 1011 Write 1 Done Done Sent false
174102024-01-24T19:13:05.408Z 13 Acked 1012 Read 10 Done Done Sent false
174112024-01-24T19:13:05.408Z 14 Acked 1013 Read 2 Done Done Sent false
174122024-01-24T19:13:05.408Z 15 Acked 1014 Read 10 Done Done Sent false
174132024-01-24T19:13:05.408Z 16 Acked 1015 Write 3 Done Done Sent false
174142024-01-24T19:13:05.408Z 17 Acked 1016 Write 7 Done Done Sent false
174152024-01-24T19:13:05.409Z 18 Acked 1017 Write 7 Done Done Sent false
174162024-01-24T19:13:05.409Z 19 Acked 1018 Read 7 Done Done Sent false
174172024-01-24T19:13:05.409Z 20 Acked 1019 Write 5 Done Done Sent false
174182024-01-24T19:13:05.409Z 21 Acked 1020 Write 3 Done Done Sent false
174192024-01-24T19:13:05.409Z 22 Acked 1021 Read 2 Done Done Sent false
174202024-01-24T19:13:05.409Z 23 Acked 1022 Flush 0 Done Done Sent false
174212024-01-24T19:13:05.409Z 24 Acked 1023 Write 9 Done Done Sent false
174222024-01-24T19:13:05.409Z 25 Acked 1024 Write 2 Done Done Sent false
174232024-01-24T19:13:05.409Z 26 Acked 1025 Read 9 Done Done Sent false
174242024-01-24T19:13:05.409Z 27 Acked 1026 Read 2 Done Done Sent false
174252024-01-24T19:13:05.410Z 28 Acked 1027 Write 3 Done Done Sent false
174262024-01-24T19:13:05.410Z 29 Acked 1028 Write 8 Sent Sent Sent false
174272024-01-24T19:13:05.410Z 30 Acked 1029 Write 4 Sent Sent Sent false
174282024-01-24T19:13:05.410Z STATES DS:0 DS:1 DS:2 TOTAL
174292024-01-24T19:13:05.410Z New 0 0 0 0
174302024-01-24T19:13:05.410Z Sent 2 2 28 32
174312024-01-24T19:13:05.410Z Done 28 28 2 58
174322024-01-24T19:13:05.410Z Skipped 0 0 0 0
174332024-01-24T19:13:05.410Z Error 0 0 0 0
174342024-01-24T19:13:05.410ZLast Flush: 1022 1022 0
174352024-01-24T19:13:05.410ZDownstairs last five completed:
174362024-01-24T19:13:05.411ZUpstairs last five completed: 30 29 28 27 26
174372024-01-24T19:13:05.411ZWrote out file "/tmp/test_repair_verify.data"
174382024-01-24T19:13:05.411Z
174392024-01-24T19:13:05.411ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174402024-01-24T19:13:05.435ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174412024-01-24T19:13:05.435Z 0 000-019 162 162 150 284 284 261 F F F <---
174422024-01-24T19:13:05.435Z 1 020-039 159 159 159 279 279 279 T T F <---
174432024-01-24T19:13:05.435Z 2 040-059 159 159 159 279 279 279 T T F <---
174442024-01-24T19:13:05.435Z 3 060-079 162 162 156 282 282 269 F F F <---
174452024-01-24T19:13:05.435Z 4 080-099 162 162 152 282 282 263 F F F <---
174462024-01-24T19:13:05.435Z 5 100-119 160 160 160 280 280 280 F F F
174472024-01-24T19:13:05.435Z 6 120-139 160 160 160 280 280 280 T T F <---
174482024-01-24T19:13:05.436Z 7 140-159 159 159 159 279 279 279 T T F <---
174492024-01-24T19:13:05.436Z 8 160-179 162 162 149 284 284 259 F F T <---
174502024-01-24T19:13:05.436Z 9 180-199 154 154 154 265 265 265 F F F
174512024-01-24T19:13:05.436Z 10 200-219 153 153 153 264 264 264 F F F
174522024-01-24T19:13:05.436Z 11 220-239 160 160 160 280 280 280 F F F
174532024-01-24T19:13:05.436Z 12 240-259 162 162 156 284 284 270 F F F <---
174542024-01-24T19:13:05.436Z 13 260-279 161 161 161 281 281 281 F F F
174552024-01-24T19:13:05.436Z 14 280-299 162 162 156 284 284 271 F F F <---
174562024-01-24T19:13:05.436Z 15 300-319 159 159 159 279 279 279 F F F
174572024-01-24T19:13:05.436Z 16 320-339 162 162 161 284 284 281 F F F <---
174582024-01-24T19:13:05.436Z 17 340-359 162 162 153 284 284 264 F F F <---
174592024-01-24T19:13:05.436Z 18 360-379 156 156 156 270 270 270 F F F
174602024-01-24T19:13:05.436Z 19 380-399 160 160 160 280 280 280 F F F
174612024-01-24T19:13:05.437Z 20 400-419 160 160 160 280 280 280 T T F <---
174622024-01-24T19:13:05.437Z 21 420-439 159 159 159 279 279 279 F F F
174632024-01-24T19:13:05.437Z 22 440-459 159 159 159 279 279 279 T T F <---
174642024-01-24T19:13:05.437Z 23 460-479 158 158 158 273 273 273 F F F
174652024-01-24T19:13:05.438Z 24 480-499 152 152 152 263 263 263 F F F
174662024-01-24T19:13:05.438Z 25 500-519 162 162 158 282 282 276 T T F <---
174672024-01-24T19:13:05.438Z 26 520-539 162 162 161 284 284 281 F F F <---
174682024-01-24T19:13:05.438Z 27 540-559 162 162 161 284 284 281 F F F <---
174692024-01-24T19:13:05.438Z 28 560-579 162 162 161 282 282 281 F F T <---
174702024-01-24T19:13:05.438Z 29 580-599 161 161 161 281 281 281 F F F
174712024-01-24T19:13:05.438ZMax gen: 162, Max flush: 284
174722024-01-24T19:13:05.438ZError: Difference in extent metadata found!
174732024-01-24T19:13:05.438ZA Difference in extent metadata is expected here
174742024-01-24T19:13:05.438ZOn loop 81
174752024-01-24T19:13:05.438Z
174762024-01-24T19:13:05.438ZVerifying data now
174772024-01-24T19:13:10.925ZNo change after verify
174782024-01-24T19:13:10.925ZLoop: 81 Downstairs dump after verify (and repair):
174792024-01-24T19:13:10.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174802024-01-24T19:13:10.953Z 0 000-019 162 162 162 284 284 284 F F F
174812024-01-24T19:13:10.954Z 1 020-039 163 163 163 285 285 285 F F F
174822024-01-24T19:13:10.954Z 2 040-059 163 163 163 285 285 285 F F F
174832024-01-24T19:13:10.954Z 3 060-079 162 162 162 282 282 282 F F F
174842024-01-24T19:13:10.971Z 4 080-099 162 162 162 282 282 282 F F F
174852024-01-24T19:13:10.971Z 5 100-119 160 160 160 280 280 280 F F F
174862024-01-24T19:13:10.971Z 6 120-139 163 163 163 285 285 285 F F F
174872024-01-24T19:13:10.971Z 7 140-159 163 163 163 285 285 285 F F F
174882024-01-24T19:13:10.972Z 8 160-179 162 162 162 284 284 284 F F F
174892024-01-24T19:13:10.972Z 9 180-199 154 154 154 265 265 265 F F F
174902024-01-24T19:13:10.972Z 10 200-219 153 153 153 264 264 264 F F F
174912024-01-24T19:13:10.972Z 11 220-239 160 160 160 280 280 280 F F F
174922024-01-24T19:13:10.972Z 12 240-259 162 162 162 284 284 284 F F F
174932024-01-24T19:13:10.972Z 13 260-279 161 161 161 281 281 281 F F F
174942024-01-24T19:13:10.972Z 14 280-299 162 162 162 284 284 284 F F F
174952024-01-24T19:13:10.972Z 15 300-319 159 159 159 279 279 279 F F F
174962024-01-24T19:13:10.972Z 16 320-339 162 162 162 284 284 284 F F F
174972024-01-24T19:13:10.972Z 17 340-359 162 162 162 284 284 284 F F F
174982024-01-24T19:13:10.972Z 18 360-379 156 156 156 270 270 270 F F F
174992024-01-24T19:13:10.972Z 19 380-399 160 160 160 280 280 280 F F F
175002024-01-24T19:13:10.972Z 20 400-419 163 163 163 285 285 285 F F F
175012024-01-24T19:13:10.972Z 21 420-439 159 159 159 279 279 279 F F F
175022024-01-24T19:13:10.972Z 22 440-459 163 163 163 285 285 285 F F F
175032024-01-24T19:13:10.972Z 23 460-479 158 158 158 273 273 273 F F F
175042024-01-24T19:13:10.974Z 24 480-499 152 152 152 263 263 263 F F F
175052024-01-24T19:13:10.974Z 25 500-519 163 163 163 285 285 285 F F F
175062024-01-24T19:13:10.974Z 26 520-539 162 162 162 284 284 284 F F F
175072024-01-24T19:13:10.974Z 27 540-559 162 162 162 284 284 284 F F F
175082024-01-24T19:13:10.974Z 28 560-579 162 162 162 282 282 282 F F F
175092024-01-24T19:13:10.975Z 29 580-599 161 161 161 281 281 281 F F F
175102024-01-24T19:13:10.975ZMax gen: 163, Max flush: 285
175112024-01-24T19:13:10.975Z
175122024-01-24T19:13:10.975ZBegin loop 82
175132024-01-24T19:13:10.975ZDownstairs to restart: 0
175142024-01-24T19:13:10.984Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.659959993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785}
175152024-01-24T19:13:10.984Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660441677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785}
175162024-01-24T19:13:10.988Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660511379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785}
175172024-01-24T19:13:10.988Z{"msg":"Crucible 10420585-a45d-4ab4-8126-7719aceaaaf0 has session id: bf86a5de-b8a8-40b5-9737-006874a10be1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660585337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175182024-01-24T19:13:10.988Z{"msg":"Crucible runtime is spawned
175192024-01-24T19:13:10.988ZUpstairs opts: Upstairs UUID: 10420585-a45d-4ab4-8126-7719aceaaaf0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660786708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175202024-01-24T19:13:11.008Z{"msg":"Crucible stats registered with UUID: 10420585-a45d-4ab4-8126-7719aceaaaf0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660847335Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175212024-01-24T19:13:11.008Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.660872959Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785}
175222024-01-24T19:13:11.008Z{"msg":"10420585-a45d-4ab4-8126-7719aceaaaf0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661094321Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175232024-01-24T19:13:11.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661181102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175242024-01-24T19:13:11.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661209262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175252024-01-24T19:13:11.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661274603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175262024-01-24T19:13:11.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661321488Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175272024-01-24T19:13:11.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661394791Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175282024-01-24T19:13:11.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661438412Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175292024-01-24T19:13:11.008Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661477886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175302024-01-24T19:13:11.008Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661517511Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175312024-01-24T19:13:11.008Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.661612977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175322024-01-24T19:13:11.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.66187775Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175332024-01-24T19:13:11.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.662173662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175342024-01-24T19:13:11.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.662362163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"io task","client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175352024-01-24T19:13:11.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663065924Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175362024-01-24T19:13:11.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663229095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175372024-01-24T19:13:11.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663265617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175382024-01-24T19:13:11.009Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663303583Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175392024-01-24T19:13:11.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663345896Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175402024-01-24T19:13:11.009Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.663435284Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175412024-01-24T19:13:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.664337542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175422024-01-24T19:13:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.664552015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175432024-01-24T19:13:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.664812425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175442024-01-24T19:13:11.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665417138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175452024-01-24T19:13:11.016Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665503804Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175462024-01-24T19:13:11.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665538384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175472024-01-24T19:13:11.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665709981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175482024-01-24T19:13:11.016Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665798089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175492024-01-24T19:13:11.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665877292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175502024-01-24T19:13:11.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.665947864Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175512024-01-24T19:13:11.016Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666023679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175522024-01-24T19:13:11.016Z{"msg":"[0]R flush_numbers[0..12]: [284, 285, 285, 282, 282, 280, 285, 285, 284, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666098619Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175532024-01-24T19:13:11.016Z{"msg":"[0]R generation[0..12]: [162, 163, 163, 162, 162, 160, 163, 163, 162, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.6661724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175542024-01-24T19:13:11.016Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
175552024-01-24T19:13:11.016Z,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.6662506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175562024-01-24T19:13:11.016Z{"msg":"[1]R flush_numbers[0..12]: [284, 285, 285, 282, 282, 280, 285, 285, 284, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666326137Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175572024-01-24T19:13:11.016Z{"msg":"[1]R generation[0..12]: [162, 163, 163, 162, 162, 160, 163, 163, 162, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666406316Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175582024-01-24T19:13:11.016Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666485059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175592024-01-24T19:13:11.016Z{"msg":"[2]R flush_numbers[0..12]: [284, 285, 285, 282, 282, 280, 285, 285, 284, 265, 264, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666558811Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175602024-01-24T19:13:11.016Z{"msg":"[2]R generation[0..12]: [162, 163, 163, 162, 162, 160, 163, 163, 162, 154, 153, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.66660541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175612024-01-24T19:13:11.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666682155Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175622024-01-24T19:13:11.017Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666756267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175632024-01-24T19:13:11.017Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666830059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175642024-01-24T19:13:11.017Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
175652024-01-24T19:13:11.017Z,"time":"2024-01-24T19:13:09.666908102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175662024-01-24T19:13:11.017Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.666987349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"":"downstairs","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175672024-01-24T19:13:11.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667052867Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175682024-01-24T19:13:11.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667078161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"0","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175692024-01-24T19:13:11.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667121233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"1","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175702024-01-24T19:13:11.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667146161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"client":"2","":"downstairs","session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175712024-01-24T19:13:11.018Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667189385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175722024-01-24T19:13:11.018Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667213042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175732024-01-24T19:13:11.018Z{"msg":"10420585-a45d-4ab4-8126-7719aceaaaf0 is now active with session: bf86a5de-b8a8-40b5-9737-006874a10be1","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667255553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175742024-01-24T19:13:11.018Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667278569Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785,"session_id":"bf86a5de-b8a8-40b5-9737-006874a10be1"}
175752024-01-24T19:13:11.018Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:09.667320038Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1785}
175762024-01-24T19:13:11.018ZLoading write count information from file "/tmp/test_repair_verify.data"
175772024-01-24T19:13:11.018ZRun Repair workload
175782024-01-24T19:13:11.018Z01/30 Write block 359 len 12288 data: 11 11 16
175792024-01-24T19:13:11.018Z02/30 Read block 34 len 32768
175802024-01-24T19:13:11.019Z03/30 Write block 537 len 8192 data: 14 13
175812024-01-24T19:13:11.019Z04/30 Read block 204 len 8192
175822024-01-24T19:13:11.019Z05/30 Flush
175832024-01-24T19:13:11.019Z06/30 Write block 294 len 24576 data: 14 12 14 12 14 13
175842024-01-24T19:13:11.019Z07/30 Read block 152 len 12288
175852024-01-24T19:13:11.019Z08/30 Read block 43 len 24576
175862024-01-24T19:13:11.019Z09/30 Write block 19 len 12288 data: 9 9 12
175872024-01-24T19:13:11.019Z10/30 Read block 361 len 20480
175882024-01-24T19:13:11.019Z11/30 Read block 360 len 24576
175892024-01-24T19:13:11.020Z12/30 Read block 578 len 32768
175902024-01-24T19:13:11.023Z13/30 Flush
175912024-01-24T19:13:11.026Z14/30 Write block 165 len 8192 data: 20 18
175922024-01-24T19:13:11.026Z15/30 Read block 223 len 16384
175932024-01-24T19:13:11.026Z16/30 Read block 332 len 32768
175942024-01-24T19:13:11.026Z17/30 Read block 9 len 16384
175952024-01-24T19:13:11.030Z18/30 Read block 170 len 24576
175962024-01-24T19:13:11.030Z19/30 Read block 458 len 12288
175972024-01-24T19:13:11.030Z20/30 Write block 561 len 8192 data: 14 11
175982024-01-24T19:13:11.030Z21/30 Read block 535 len 4096
175992024-01-24T19:13:11.030Z22/30 Write block 128 len 8192 data: 7 8
176002024-01-24T19:13:11.030Z23/30 Read block 217 len 16384
176012024-01-24T19:13:11.033Z24/30 Read block 420 len 40960
176022024-01-24T19:13:11.036Z25/30 Read block 14 len 40960
176032024-01-24T19:13:11.036Z26/30 Read block 305 len 24576
176042024-01-24T19:13:11.040Z27/30 Read block 394 len 20480
176052024-01-24T19:13:11.040Z28/30 Write block 199 len 28672 data: 8 8 8 6 7 9 9
176062024-01-24T19:13:11.043Z29/30 Write block 5 len 28672 data: 8 10 11 13 10 10 9
176072024-01-24T19:13:11.043Z30/30 Read block 571 len 24576
176082024-01-24T19:13:11.047Z----------------------------------------------------------------
176092024-01-24T19:13:11.047Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
176102024-01-24T19:13:11.047ZStates: Active Active Active
176112024-01-24T19:13:11.047ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176122024-01-24T19:13:11.047Z 1 Acked 1000 Write 3 Sent Done Done false
176132024-01-24T19:13:11.051Z 2 Acked 1001 Read 8 Sent Done Done false
176142024-01-24T19:13:11.051Z 3 Acked 1002 Write 2 Sent Done Done false
176152024-01-24T19:13:11.051Z 4 Acked 1003 Read 2 Sent Done Done false
176162024-01-24T19:13:11.051Z 5 Acked 1004 Flush 0 Sent Done Done false
176172024-01-24T19:13:11.051Z 6 Acked 1005 Write 6 Sent Done Done false
176182024-01-24T19:13:11.051Z 7 Acked 1006 Read 3 Sent Done Done false
176192024-01-24T19:13:11.051Z 8 Acked 1007 Read 6 Sent Done Done false
176202024-01-24T19:13:11.052Z 9 Acked 1008 Write 3 Sent Done Done false
176212024-01-24T19:13:11.052Z 10 Acked 1009 Read 5 Sent Done Done false
176222024-01-24T19:13:11.052Z 11 Acked 1010 Read 6 Sent Done Done false
176232024-01-24T19:13:11.052Z 12 Acked 1011 Read 8 Sent Done Done false
176242024-01-24T19:13:11.052Z 13 Acked 1012 Flush 0 Sent Done Done false
176252024-01-24T19:13:11.052Z 14 Acked 1013 Write 2 Sent Done Done false
176262024-01-24T19:13:11.052Z 15 Acked 1014 Read 4 Sent Done Done false
176272024-01-24T19:13:11.052Z 16 Acked 1015 Read 8 Sent Done Done false
176282024-01-24T19:13:11.052Z 17 Acked 1016 Read 4 Sent Done Done false
176292024-01-24T19:13:11.052Z 18 Acked 1017 Read 6 Sent Done Done false
176302024-01-24T19:13:11.052Z 19 Acked 1018 Read 3 Sent Done Done false
176312024-01-24T19:13:11.052Z 20 Acked 1019 Write 2 Sent Done Done false
176322024-01-24T19:13:11.052Z 21 Acked 1020 Read 1 Sent Done Done false
176332024-01-24T19:13:11.052Z 22 Acked 1021 Write 2 Sent Done Done false
176342024-01-24T19:13:11.052Z 23 Acked 1022 Read 4 Sent Done Done false
176352024-01-24T19:13:11.052Z 24 Acked 1023 Read 10 Sent Done Done false
176362024-01-24T19:13:11.053Z 25 Acked 1024 Read 10 Sent Done Done false
176372024-01-24T19:13:11.053Z 26 Acked 1025 Read 6 Sent Done Done false
176382024-01-24T19:13:11.053Z 27 Acked 1026 Read 5 Sent Done Done false
176392024-01-24T19:13:11.053Z 28 Acked 1027 Write 7 Sent Done Done false
176402024-01-24T19:13:11.053Z 29 Acked 1028 Write 7 Sent Done Sent false
176412024-01-24T19:13:11.053Z 30 Acked 1029 Read 6 Sent Done Sent false
176422024-01-24T19:13:11.053Z STATES DS:0 DS:1 DS:2 TOTAL
176432024-01-24T19:13:11.053Z New 0 0 0 0
176442024-01-24T19:13:11.053Z Sent 30 0 2 32
176452024-01-24T19:13:11.053Z Done 0 30 28 58
176462024-01-24T19:13:11.053Z Skipped 0 0 0 0
176472024-01-24T19:13:11.053Z Error 0 0 0 0
176482024-01-24T19:13:11.053ZLast Flush: 0 1012 1012
176492024-01-24T19:13:11.054ZDownstairs last five completed:
176502024-01-24T19:13:11.054ZUpstairs last five completed: 30 29 28 27 26
176512024-01-24T19:13:11.054ZWrote out file "/tmp/test_repair_verify.data"
176522024-01-24T19:13:11.057Z
176532024-01-24T19:13:11.057ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
176542024-01-24T19:13:11.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176552024-01-24T19:13:11.086Z 0 000-019 162 164 164 284 287 287 F T T <---
176562024-01-24T19:13:11.090Z 1 020-039 163 164 164 285 287 287 F F F <---
176572024-01-24T19:13:11.090Z 2 040-059 163 163 163 285 285 285 F F F
176582024-01-24T19:13:11.090Z 3 060-079 162 162 162 282 282 282 F F F
176592024-01-24T19:13:11.090Z 4 080-099 162 162 162 282 282 282 F F F
176602024-01-24T19:13:11.091Z 5 100-119 160 160 160 280 280 280 F F F
176612024-01-24T19:13:11.091Z 6 120-139 163 163 163 285 285 285 F T T <---
176622024-01-24T19:13:11.091Z 7 140-159 163 163 163 285 285 285 F F F
176632024-01-24T19:13:11.091Z 8 160-179 162 162 162 284 284 284 F T T <---
176642024-01-24T19:13:11.091Z 9 180-199 154 154 154 265 265 265 F T T <---
176652024-01-24T19:13:11.092Z 10 200-219 153 153 153 264 264 264 F T T <---
176662024-01-24T19:13:11.092Z 11 220-239 160 160 160 280 280 280 F F F
176672024-01-24T19:13:11.092Z 12 240-259 162 162 162 284 284 284 F F F
176682024-01-24T19:13:11.092Z 13 260-279 161 161 161 281 281 281 F F F
176692024-01-24T19:13:11.092Z 14 280-299 162 164 164 284 287 287 F F F <---
176702024-01-24T19:13:11.092Z 15 300-319 159 159 159 279 279 279 F F F
176712024-01-24T19:13:11.092Z 16 320-339 162 162 162 284 284 284 F F F
176722024-01-24T19:13:11.092Z 17 340-359 162 164 164 284 286 286 F F F <---
176732024-01-24T19:13:11.092Z 18 360-379 156 164 164 270 286 286 F F F <---
176742024-01-24T19:13:11.092Z 19 380-399 160 160 160 280 280 280 F F F
176752024-01-24T19:13:11.092Z 20 400-419 163 163 163 285 285 285 F F F
176762024-01-24T19:13:11.092Z 21 420-439 159 159 159 279 279 279 F F F
176772024-01-24T19:13:11.092Z 22 440-459 163 163 163 285 285 285 F F F
176782024-01-24T19:13:11.092Z 23 460-479 158 158 158 273 273 273 F F F
176792024-01-24T19:13:11.092Z 24 480-499 152 152 152 263 263 263 F F F
176802024-01-24T19:13:11.092Z 25 500-519 163 163 163 285 285 285 F F F
176812024-01-24T19:13:11.092Z 26 520-539 162 164 164 284 286 286 F F F <---
176822024-01-24T19:13:11.092Z 27 540-559 162 162 162 284 284 284 F F F
176832024-01-24T19:13:11.092Z 28 560-579 162 162 162 282 282 282 F T T <---
176842024-01-24T19:13:11.093Z 29 580-599 161 161 161 281 281 281 F F F
176852024-01-24T19:13:11.094ZMax gen: 164, Max flush: 287
176862024-01-24T19:13:11.094ZError: Difference in extent metadata found!
176872024-01-24T19:13:11.094ZA Difference in extent metadata is expected here
176882024-01-24T19:13:11.094ZOn loop 82
176892024-01-24T19:13:11.094Z
176902024-01-24T19:13:11.097ZVerifying data now
176912024-01-24T19:13:16.104ZNo change after verify
176922024-01-24T19:13:16.104ZLoop: 82 Downstairs dump after verify (and repair):
176932024-01-24T19:13:16.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176942024-01-24T19:13:16.135Z 0 000-019 165 165 165 288 288 288 F F F
176952024-01-24T19:13:16.146Z 1 020-039 164 164 164 287 287 287 F F F
176962024-01-24T19:13:16.146Z 2 040-059 163 163 163 285 285 285 F F F
176972024-01-24T19:13:16.146Z 3 060-079 162 162 162 282 282 282 F F F
176982024-01-24T19:13:16.146Z 4 080-099 162 162 162 282 282 282 F F F
176992024-01-24T19:13:16.146Z 5 100-119 160 160 160 280 280 280 F F F
177002024-01-24T19:13:16.146Z 6 120-139 165 165 165 288 288 288 F F F
177012024-01-24T19:13:16.146Z 7 140-159 163 163 163 285 285 285 F F F
177022024-01-24T19:13:16.146Z 8 160-179 165 165 165 288 288 288 F F F
177032024-01-24T19:13:16.146Z 9 180-199 165 165 165 288 288 288 F F F
177042024-01-24T19:13:16.147Z 10 200-219 165 165 165 288 288 288 F F F
177052024-01-24T19:13:16.147Z 11 220-239 160 160 160 280 280 280 F F F
177062024-01-24T19:13:16.147Z 12 240-259 162 162 162 284 284 284 F F F
177072024-01-24T19:13:16.147Z 13 260-279 161 161 161 281 281 281 F F F
177082024-01-24T19:13:16.147Z 14 280-299 164 164 164 287 287 287 F F F
177092024-01-24T19:13:16.147Z 15 300-319 159 159 159 279 279 279 F F F
177102024-01-24T19:13:16.147Z 16 320-339 162 162 162 284 284 284 F F F
177112024-01-24T19:13:16.147Z 17 340-359 164 164 164 286 286 286 F F F
177122024-01-24T19:13:16.147Z 18 360-379 164 164 164 286 286 286 F F F
177132024-01-24T19:13:16.147Z 19 380-399 160 160 160 280 280 280 F F F
177142024-01-24T19:13:16.147Z 20 400-419 163 163 163 285 285 285 F F F
177152024-01-24T19:13:16.147Z 21 420-439 159 159 159 279 279 279 F F F
177162024-01-24T19:13:16.147Z 22 440-459 163 163 163 285 285 285 F F F
177172024-01-24T19:13:16.148Z 23 460-479 158 158 158 273 273 273 F F F
177182024-01-24T19:13:16.148Z 24 480-499 152 152 152 263 263 263 F F F
177192024-01-24T19:13:16.148Z 25 500-519 163 163 163 285 285 285 F F F
177202024-01-24T19:13:16.148Z 26 520-539 164 164 164 286 286 286 F F F
177212024-01-24T19:13:16.148Z 27 540-559 162 162 162 284 284 284 F F F
177222024-01-24T19:13:16.148Z 28 560-579 165 165 165 288 288 288 F F F
177232024-01-24T19:13:16.148Z 29 580-599 161 161 161 281 281 281 F F F
177242024-01-24T19:13:16.148ZMax gen: 165, Max flush: 288
177252024-01-24T19:13:16.148Z
177262024-01-24T19:13:16.148ZBegin loop 83
177272024-01-24T19:13:16.148ZDownstairs to restart: 2
177282024-01-24T19:13:16.158Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.834689891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793}
177292024-01-24T19:13:16.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835156187Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793}
177302024-01-24T19:13:16.163Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835212954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793}
177312024-01-24T19:13:16.163Z{"msg":"Crucible 4b4c9de2-fd51-4a66-be6f-7a881dd74d2f has session id: 3c2bf4a1-8138-4e44-a278-0a3f9765c34e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835373533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal"Crucible runtime is spawned
177322024-01-24T19:13:16.163Z,"pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177332024-01-24T19:13:16.163Z{"msg":"Upstairs opts: Upstairs UUID: 4b4c9de2-fd51-4a66-be6f-7a881dd74d2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835540494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177342024-01-24T19:13:16.164Z{"msg":"Crucible stats registered with UUID: 4b4c9de2-fd51-4a66-be6f-7a881dd74d2f","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835587059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177352024-01-24T19:13:16.164Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835636021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793}
177362024-01-24T19:13:16.164Z{"msg":"4b4c9de2-fd51-4a66-be6f-7a881dd74d2f active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835796547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177372024-01-24T19:13:16.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.835855894Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177382024-01-24T19:13:16.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.83588336Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177392024-01-24T19:13:16.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.83590655Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177402024-01-24T19:13:16.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836006763Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177412024-01-24T19:13:16.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836212012Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177422024-01-24T19:13:16.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836314071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177432024-01-24T19:13:16.164Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836360789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177442024-01-24T19:13:16.167Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.83645938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177452024-01-24T19:13:16.167Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836506806Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177462024-01-24T19:13:16.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836551129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177472024-01-24T19:13:16.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836625546Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177482024-01-24T19:13:16.167Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.836668541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177492024-01-24T19:13:16.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.837468852Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177502024-01-24T19:13:16.167Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.83752301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177512024-01-24T19:13:16.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.837546571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177522024-01-24T19:13:16.167Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.837731018Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177532024-01-24T19:13:16.176Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.838674098Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177542024-01-24T19:13:16.176Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.838940733Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177552024-01-24T19:13:16.176Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.839053824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177562024-01-24T19:13:16.176Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.83915766Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177572024-01-24T19:13:16.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.839567318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177582024-01-24T19:13:16.176Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.839630359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177592024-01-24T19:13:16.176Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.839736244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177602024-01-24T19:13:16.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.840064708Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177612024-01-24T19:13:16.176Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.840149971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177622024-01-24T19:13:16.176Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.840181031Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177632024-01-24T19:13:16.176Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.84023505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177642024-01-24T19:13:16.177Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841008304Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177652024-01-24T19:13:16.177Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841069528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177662024-01-24T19:13:16.178Z{"msg":"[0]R flush_numbers[0..12]: [288, 287, 285, 282, 282, 280, 288, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841119232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177672024-01-24T19:13:16.178Z{"msg":"[0]R generation[0..12]: [165, 164, 163, 162, 162, 160, 165, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.8411463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177682024-01-24T19:13:16.178Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.84119895Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177692024-01-24T19:13:16.178Z{"msg":"[1]R flush_numbers[0..12]: [288, 287, 285, 282, 282, 280, 288, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841256144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177702024-01-24T19:13:16.178Z{"msg":"[1]R generation[0..12]: [165, 164, 163, 162, 162, 160, 165, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841283939Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177712024-01-24T19:13:16.178Z{"msg":"Wait for a query_work_queue command to finish before sending IO
177722024-01-24T19:13:16.178Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841347671Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177732024-01-24T19:13:16.178Z{"msg":"[2]R flush_numbers[0..12]: [288, 287, 285, 282, 282, 280, 288, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841420417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177742024-01-24T19:13:16.178Z{"msg":"[2]R generation[0..12]: [165, 164, 163, 162, 162, 160, 165, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841454797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177752024-01-24T19:13:16.178Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841514549Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177762024-01-24T19:13:16.178Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.84159281Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177772024-01-24T19:13:16.178Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841621105Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177782024-01-24T19:13:16.178Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.84172499Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177792024-01-24T19:13:16.178Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841752362Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"":"downstairs","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177802024-01-24T19:13:16.178Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841774325Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177812024-01-24T19:13:16.178Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841833119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"0","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177822024-01-24T19:13:16.178Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841860604Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"1","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177832024-01-24T19:13:16.179Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841919361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"client":"2","":"downstairs","session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177842024-01-24T19:13:16.181Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.841951982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177852024-01-24T19:13:16.181Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.842022453Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177862024-01-24T19:13:16.181Z{"msg":"4b4c9de2-fd51-4a66-be6f-7a881dd74d2f is now active with session: 3c2bf4a1-8138-4e44-a278-0a3f9765c34e","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.842050387Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177872024-01-24T19:13:16.181Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.842108151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793,"session_id":"3c2bf4a1-8138-4e44-a278-0a3f9765c34e"}
177882024-01-24T19:13:16.181Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:14.842139027Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1793}
177892024-01-24T19:13:16.181ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
177902024-01-24T19:13:16.181ZLoading write count information from file "/tmp/test_repair_verify.data"
177912024-01-24T19:13:16.181ZRun Repair workload
177922024-01-24T19:13:16.181Z01/30 Write block 122 len 8192 data: 13 12
177932024-01-24T19:13:16.181Z02/30 Read block 577 len 28672
177942024-01-24T19:13:16.181Z03/30 Read block 464 len 16384
177952024-01-24T19:13:16.185Z04/30 Read block 346 len 16384
177962024-01-24T19:13:16.185Z05/30 Read block 156 len 36864
177972024-01-24T19:13:16.185Z06/30 Read block 261 len 12288
177982024-01-24T19:13:16.185Z07/30 Read block 80 len 4096
177992024-01-24T19:13:16.185Z08/30 Write block 373 len 32768 data: 12 11 15 13 13 9 10 13
178002024-01-24T19:13:16.185Z09/30 Read block 238 len 4096
178012024-01-24T19:13:16.185Z10/30 Read block 156 len 32768
178022024-01-24T19:13:16.186Z11/30 Flush
178032024-01-24T19:13:16.189Z12/30 Write block 30 len 20480 data: 14 16 14 14 14
178042024-01-24T19:13:16.189Z13/30 Write block 574 len 24576 data: 13 14 14 9 11 11
178052024-01-24T19:13:16.189Z14/30 Read block 59 len 24576
178062024-01-24T19:13:16.194Z15/30 Write block 367 len 12288 data: 16 15 15
178072024-01-24T19:13:16.194Z16/30 Write block 272 len 8192 data: 14 16
178082024-01-24T19:13:16.194Z17/30 Read block 546 len 8192
178092024-01-24T19:13:16.198Z18/30 Write block 29 len 4096 data: 16
178102024-01-24T19:13:16.199Z19/30 Flush
178112024-01-24T19:13:16.199Z20/30 Write block 50 len 28672 data: 16 15 14 13 12 12 13
178122024-01-24T19:13:16.202Z21/30 Flush
178132024-01-24T19:13:16.202Z22/30 Write block 48 len 24576 data: 14 15 17 16 15 14
178142024-01-24T19:13:16.205Z23/30 Read block 432 len 28672
178152024-01-24T19:13:16.205Z24/30 Write block 264 len 16384 data: 12 13 11 13
178162024-01-24T19:13:16.205Z25/30 Read block 366 len 24576
178172024-01-24T19:13:16.209Z26/30 Read block 76 len 32768
178182024-01-24T19:13:16.213Z27/30 Write block 453 len 24576 data: 11 10 9 12 11 11
178192024-01-24T19:13:16.213Z28/30 Write block 105 len 12288 data: 15 16 17
178202024-01-24T19:13:16.214Z29/30 Write block 314 len 4096 data: 15
178212024-01-24T19:13:16.214Z30/30 Read block 526 len 32768
178222024-01-24T19:13:16.217Z----------------------------------------------------------------
178232024-01-24T19:13:16.217Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
178242024-01-24T19:13:16.217ZStates: Active Active Active
178252024-01-24T19:13:16.217ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178262024-01-24T19:13:16.217Z 1 Acked 1000 Write 2 Done Done Sent false
178272024-01-24T19:13:16.217Z 2 Acked 1001 Read 7 Done Done Sent false
178282024-01-24T19:13:16.217Z 3 Acked 1002 Read 4 Done Done Sent false
178292024-01-24T19:13:16.217Z 4 Acked 1003 Read 4 Done Done Sent false
178302024-01-24T19:13:16.218Z 5 Acked 1004 Read 9 Done Done Sent false
178312024-01-24T19:13:16.218Z 6 Acked 1005 Read 3 Done Done Sent false
178322024-01-24T19:13:16.218Z 7 Acked 1006 Read 1 Done Done Sent false
178332024-01-24T19:13:16.218Z 8 Acked 1007 Write 8 Done Done Sent false
178342024-01-24T19:13:16.218Z 9 Acked 1008 Read 1 Done Done Sent false
178352024-01-24T19:13:16.218Z 10 Acked 1009 Read 8 Done Done Sent false
178362024-01-24T19:13:16.218Z 11 Acked 1010 Flush 0 Done Done Sent false
178372024-01-24T19:13:16.218Z 12 Acked 1011 Write 5 Done Done Sent false
178382024-01-24T19:13:16.218Z 13 Acked 1012 Write 6 Done Done Sent false
178392024-01-24T19:13:16.218Z 14 Acked 1013 Read 6 Done Done Sent false
178402024-01-24T19:13:16.218Z 15 Acked 1014 Write 3 Done Done Sent false
178412024-01-24T19:13:16.218Z 16 Acked 1015 Write 2 Done Done Sent false
178422024-01-24T19:13:16.218Z 17 Acked 1016 Read 2 Done Done Sent false
178432024-01-24T19:13:16.218Z 18 Acked 1017 Write 1 Done Done Sent false
178442024-01-24T19:13:16.218Z 19 Acked 1018 Flush 0 Done Done Sent false
178452024-01-24T19:13:16.218Z 20 Acked 1019 Write 7 Done Done Sent false
178462024-01-24T19:13:16.218Z 21 Acked 1020 Flush 0 Done Done Sent false
178472024-01-24T19:13:16.218Z 22 Acked 1021 Write 6 Done Done Sent false
178482024-01-24T19:13:16.218Z 23 Acked 1022 Read 7 Done Done Sent false
178492024-01-24T19:13:16.218Z 24 Acked 1023 Write 4 Done Done Sent false
178502024-01-24T19:13:16.218Z 25 Acked 1024 Read 6 Done Done Sent false
178512024-01-24T19:13:16.218Z 26 Acked 1025 Read 8 Done Done Sent false
178522024-01-24T19:13:16.218Z 27 Acked 1026 Write 6 Done Done Sent false
178532024-01-24T19:13:16.219Z 28 Acked 1027 Write 3 Done Done Sent false
178542024-01-24T19:13:16.219Z 29 Acked 1028 Write 1 Done Done Sent false
178552024-01-24T19:13:16.219Z 30 Acked 1029 Read 8 Done Sent Sent false
178562024-01-24T19:13:16.219Z STATES DS:0 DS:1 DS:2 TOTAL
178572024-01-24T19:13:16.219Z New 0 0 0 0
178582024-01-24T19:13:16.219Z Sent 0 1 30 31
178592024-01-24T19:13:16.219Z Done 30 29 0 59
178602024-01-24T19:13:16.219Z Skipped 0 0 0 0
178612024-01-24T19:13:16.219Z Error 0 0 0 0
178622024-01-24T19:13:16.219ZLast Flush: 1020 1020 0
178632024-01-24T19:13:16.219ZDownstairs last five completed:
178642024-01-24T19:13:16.219ZUpstairs last five completed: 30 29 28 27 26
178652024-01-24T19:13:16.219ZWrote out file "/tmp/test_repair_verify.data"
178662024-01-24T19:13:16.224Z
178672024-01-24T19:13:16.228ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178682024-01-24T19:13:16.252ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178692024-01-24T19:13:16.252Z 0 000-019 165 165 165 288 288 288 F F F
178702024-01-24T19:13:16.256Z 1 020-039 166 166 164 290 290 287 F F F <---
178712024-01-24T19:13:16.256Z 2 040-059 166 166 163 291 291 285 T T F <---
178722024-01-24T19:13:16.256Z 3 060-079 162 162 162 282 282 282 F F F
178732024-01-24T19:13:16.256Z 4 080-099 162 162 162 282 282 282 F F F
178742024-01-24T19:13:16.256Z 5 100-119 160 160 160 280 280 280 T T F <---
178752024-01-24T19:13:16.256Z 6 120-139 166 166 165 289 289 288 F F F <---
178762024-01-24T19:13:16.256Z 7 140-159 163 163 163 285 285 285 F F F
178772024-01-24T19:13:16.257Z 8 160-179 165 165 165 288 288 288 F F F
178782024-01-24T19:13:16.257Z 9 180-199 165 165 165 288 288 288 F F F
178792024-01-24T19:13:16.257Z 10 200-219 165 165 165 288 288 288 F F F
178802024-01-24T19:13:16.257Z 11 220-239 160 160 160 280 280 280 F F F
178812024-01-24T19:13:16.257Z 12 240-259 162 162 162 284 284 284 F F F
178822024-01-24T19:13:16.257Z 13 260-279 166 166 161 290 290 281 T T F <---
178832024-01-24T19:13:16.257Z 14 280-299 164 164 164 287 287 287 F F F
178842024-01-24T19:13:16.257Z 15 300-319 159 159 159 279 279 279 T T F <---
178852024-01-24T19:13:16.257Z 16 320-339 162 162 162 284 284 284 F F F
178862024-01-24T19:13:16.257Z 17 340-359 164 164 164 286 286 286 F F F
178872024-01-24T19:13:16.257Z 18 360-379 166 166 164 290 290 286 F F F <---
178882024-01-24T19:13:16.257Z 19 380-399 166 166 160 289 289 280 F F F <---
178892024-01-24T19:13:16.257Z 20 400-419 163 163 163 285 285 285 F F F
178902024-01-24T19:13:16.257Z 21 420-439 159 159 159 279 279 279 F F F
178912024-01-24T19:13:16.257Z 22 440-459 163 163 163 285 285 285 T T F <---
178922024-01-24T19:13:16.257Z 23 460-479 158 158 158 273 273 273 F F F
178932024-01-24T19:13:16.257Z 24 480-499 152 152 152 263 263 263 F F F
178942024-01-24T19:13:16.257Z 25 500-519 163 163 163 285 285 285 F F F
178952024-01-24T19:13:16.257Z 26 520-539 164 164 164 286 286 286 F F F
178962024-01-24T19:13:16.260Z 27 540-559 162 162 162 284 284 284 F F F
178972024-01-24T19:13:16.260Z 28 560-579 166 166 165 290 290 288 F F F <---
178982024-01-24T19:13:16.260Z 29 580-599 161 161 161 281 281 281 F F F
178992024-01-24T19:13:16.260ZMax gen: 166, Max flush: 291
179002024-01-24T19:13:16.260ZError: Difference in extent metadata found!
179012024-01-24T19:13:16.264ZA Difference in extent metadata is expected here
179022024-01-24T19:13:16.264ZOn loop 83
179032024-01-24T19:13:16.264Z
179042024-01-24T19:13:16.264ZVerifying data now
179052024-01-24T19:13:21.274ZNo change after verify
179062024-01-24T19:13:21.274ZLoop: 83 Downstairs dump after verify (and repair):
179072024-01-24T19:13:21.283ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179082024-01-24T19:13:21.283Z 0 000-019 165 165 165 288 288 288 F F F
179092024-01-24T19:13:21.283Z 1 020-039 166 166 166 290 290 290 F F F
179102024-01-24T19:13:21.283Z 2 040-059 167 167 167 292 292 292 F F F
179112024-01-24T19:13:21.283Z 3 060-079 162 162 162 282 282 282 F F F
179122024-01-24T19:13:21.283Z 4 080-099 162 162 162 282 282 282 F F F
179132024-01-24T19:13:21.283Z 5 100-119 167 167 167 292 292 292 F F F
179142024-01-24T19:13:21.283Z 6 120-139 166 166 166 289 289 289 F F F
179152024-01-24T19:13:21.283Z 7 140-159 163 163 163 285 285 285 F F F
179162024-01-24T19:13:21.284Z 8 160-179 165 165 165 288 288 288 F F F
179172024-01-24T19:13:21.284Z 9 180-199 165 165 165 288 288 288 F F F
179182024-01-24T19:13:21.284Z 10 200-219 165 165 165 288 288 288 F F F
179192024-01-24T19:13:21.284Z 11 220-239 160 160 160 280 280 280 F F F
179202024-01-24T19:13:21.284Z 12 240-259 162 162 162 284 284 284 F F F
179212024-01-24T19:13:21.284Z 13 260-279 167 167 167 292 292 292 F F F
179222024-01-24T19:13:21.284Z 14 280-299 164 164 164 287 287 287 F F F
179232024-01-24T19:13:21.284Z 15 300-319 167 167 167 292 292 292 F F F
179242024-01-24T19:13:21.284Z 16 320-339 162 162 162 284 284 284 F F F
179252024-01-24T19:13:21.284Z 17 340-359 164 164 164 286 286 286 F F F
179262024-01-24T19:13:21.284Z 18 360-379 166 166 166 290 290 290 F F F
179272024-01-24T19:13:21.284Z 19 380-399 166 166 166 289 289 289 F F F
179282024-01-24T19:13:21.284Z 20 400-419 163 163 163 285 285 285 F F F
179292024-01-24T19:13:21.284Z 21 420-439 159 159 159 279 279 279 F F F
179302024-01-24T19:13:21.285Z 22 440-459 167 167 167 292 292 292 F F F
179312024-01-24T19:13:21.285Z 23 460-479 158 158 158 273 273 273 F F F
179322024-01-24T19:13:21.285Z 24 480-499 152 152 152 263 263 263 F F F
179332024-01-24T19:13:21.286Z 25 500-519 163 163 163 285 285 285 F F F
179342024-01-24T19:13:21.286Z 26 520-539 164 164 164 286 286 286 F F F
179352024-01-24T19:13:21.286Z 27 540-559 162 162 162 284 284 284 F F F
179362024-01-24T19:13:21.286Z 28 560-579 166 166 166 290 290 290 F F F
179372024-01-24T19:13:21.286Z 29 580-599 161 161 161 281 281 281 F F F
179382024-01-24T19:13:21.286ZMax gen: 167, Max flush: 292
179392024-01-24T19:13:21.286Z
179402024-01-24T19:13:21.286ZBegin loop 84
179412024-01-24T19:13:21.286ZDownstairs to restart: 2
179422024-01-24T19:13:21.286Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.924827434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801}
179432024-01-24T19:13:21.286Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.925241628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801}
179442024-01-24T19:13:21.286Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.925293587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801}
179452024-01-24T19:13:21.286Z{"msg":"Crucible 6523caa3-c016-4e99-9dc4-a12a4bd62ae8 has session id: 08afc302-a9b7-468c-91a8-5446fab9013b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.925318529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179462024-01-24T19:13:21.286Z{"msg":"Upstairs opts: Upstairs UUID: 6523caa3-c016-4e99-9dc4-a12a4bd62ae8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.92535403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179472024-01-24T19:13:21.286Z{"msg":"Crucible stats registered with UUID: 6523caa3-c016-4e99-9dc4-a12a4bd62ae8","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.925416138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179482024-01-24T19:13:21.286ZCrucible runtime is spawned
179492024-01-24T19:13:21.286Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.925657464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801}
179502024-01-24T19:13:21.286Z{"msg":"6523caa3-c016-4e99-9dc4-a12a4bd62ae8 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926014594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179512024-01-24T19:13:21.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926149763Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179522024-01-24T19:13:21.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926258487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179532024-01-24T19:13:21.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.92630633Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179542024-01-24T19:13:21.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926387073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179552024-01-24T19:13:21.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926580677Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179562024-01-24T19:13:21.287Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.926747267Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179572024-01-24T19:13:21.290Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927037358Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179582024-01-24T19:13:21.290Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927145736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179592024-01-24T19:13:21.290Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927184217Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179602024-01-24T19:13:21.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927209292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179612024-01-24T19:13:21.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927251127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179622024-01-24T19:13:21.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.927293264Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179632024-01-24T19:13:21.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.92815785Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179642024-01-24T19:13:21.290Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.928274395Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179652024-01-24T19:13:21.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.928372505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179662024-01-24T19:13:21.290Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.928424242Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179672024-01-24T19:13:21.291Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.928619059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179682024-01-24T19:13:21.291Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.928705295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179692024-01-24T19:13:21.291Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.929185222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179702024-01-24T19:13:21.291Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.929304913Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179712024-01-24T19:13:21.291Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.929753394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179722024-01-24T19:13:21.291Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930153229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179732024-01-24T19:13:21.291Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930215431Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179742024-01-24T19:13:21.291Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930242935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179752024-01-24T19:13:21.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930329776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179762024-01-24T19:13:21.293Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930460345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179772024-01-24T19:13:21.293Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930486575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179782024-01-24T19:13:21.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930528761Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179792024-01-24T19:13:21.293Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930557914Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179802024-01-24T19:13:21.293Z{"msg":"[0]R flush_numbers[0..12]: [288, 290, 292, 282, 282, 292, 289, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930630944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179812024-01-24T19:13:21.293Z{"msg":"[0]R generation[0..12]: [165, 166, 167, 162, 162, 167, 166, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930705826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179822024-01-24T19:13:21.293Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930737413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179832024-01-24T19:13:21.293Z{"msg":"[1]R flush_numbers[0..12]: [288, 290, 292, 282, 282, 292, 289, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930783532Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179842024-01-24T19:13:21.293Z{"msg":"[1]R generation[0..12]: [165, 166, 167, 162, 162, 167, 166, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930810397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179852024-01-24T19:13:21.293Z{"msg":"Wait for a query_work_queue command to finish before sending IO
179862024-01-24T19:13:21.293Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930864405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179872024-01-24T19:13:21.293Z{"msg":"[2]R flush_numbers[0..12]: [288, 290, 292, 282, 282, 292, 289, 285, 288, 288, 288, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930914683Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179882024-01-24T19:13:21.293Z{"msg":"[2]R generation[0..12]: [165, 166, 167, 162, 162, 167, 166, 163, 165, 165, 165, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930940919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179892024-01-24T19:13:21.293Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.930985647Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179902024-01-24T19:13:21.293Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931010744Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179912024-01-24T19:13:21.293Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931039807Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179922024-01-24T19:13:21.293Z{"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931084433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179932024-01-24T19:13:21.293Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.93110963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"":"downstairs","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179942024-01-24T19:13:21.294Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931180514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179952024-01-24T19:13:21.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931264921Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"0","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179962024-01-24T19:13:21.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931299072Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"1","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179972024-01-24T19:13:21.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.93132172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"client":"2","":"downstairs","session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179982024-01-24T19:13:21.294Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931343513Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
179992024-01-24T19:13:21.295Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931403014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
180002024-01-24T19:13:21.295Z{"msg":"6523caa3-c016-4e99-9dc4-a12a4bd62ae8 is now active with session: 08afc302-a9b7-468c-91a8-5446fab9013b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931430838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
180012024-01-24T19:13:21.295Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:19.931458274Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801,"session_id":"08afc302-a9b7-468c-91a8-5446fab9013b"}
180022024-01-24T19:13:21.295Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180032024-01-24T19:13:21.295Z,"time":"2024-01-24T19:13:19.931543142Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1801}
180042024-01-24T19:13:21.295ZLoading write count information from file "/tmp/test_repair_verify.data"
180052024-01-24T19:13:21.295ZRun Repair workload
180062024-01-24T19:13:21.295Z01/30 Write block 341 len 32768 data: 11 9 13 12 12 10 12 10
180072024-01-24T19:13:21.299Z02/30 Write block 514 len 16384 data: 17 16 14 19
180082024-01-24T19:13:21.300Z03/30 Read block 55 len 24576
180092024-01-24T19:13:21.300Z04/30 Write block 523 len 28672 data: 12 14 15 15 13 18 16
180102024-01-24T19:13:21.300Z05/30 Read block 504 len 16384
180112024-01-24T19:13:21.300Z06/30 Write block 55 len 4096 data: 13
180122024-01-24T19:13:21.300Z07/30 Read block 208 len 20480
180132024-01-24T19:13:21.302Z08/30 Read block 553 len 12288
180142024-01-24T19:13:21.302Z09/30 Read block 514 len 4096
180152024-01-24T19:13:21.302Z10/30 Write block 324 len 28672 data: 12 14 9 12 14 15 13
180162024-01-24T19:13:21.302Z11/30 Write block 546 len 8192 data: 15 16
180172024-01-24T19:13:21.302Z12/30 Write block 533 len 12288 data: 10 12 11
180182024-01-24T19:13:21.302Z13/30 Write block 318 len 36864 data: 13 14 13 12 12 13 13 15 10
180192024-01-24T19:13:21.302Z14/30 Write block 583 len 32768 data: 12 13 12 13 12 12 10 10
180202024-01-24T19:13:21.302Z15/30 Flush
180212024-01-24T19:13:21.303Z16/30 Write block 271 len 8192 data: 12 15
180222024-01-24T19:13:21.303Z17/30 Write block 414 len 20480 data: 9 8 8 10 12
180232024-01-24T19:13:21.303Z18/30 Read block 385 len 36864
180242024-01-24T19:13:21.303Z19/30 Read block 23 len 36864
180252024-01-24T19:13:21.303Z20/30 Flush
180262024-01-24T19:13:21.303Z21/30 Write block 434 len 4096 data: 15
180272024-01-24T19:13:21.303Z22/30 Write block 15 len 32768 data: 9 5 6 7 10 10 13 13
180282024-01-24T19:13:21.303Z23/30 Write block 509 len 4096 data: 13
180292024-01-24T19:13:21.303Z24/30 Write block 57 len 8192 data: 12 10
180302024-01-24T19:13:21.303Z25/30 Flush
180312024-01-24T19:13:21.303Z26/30 Flush
180322024-01-24T19:13:21.303Z27/30 Write block 210 len 28672 data: 13 11 11 10 7 7 10
180332024-01-24T19:13:21.303Z28/30 Write block 94 len 4096 data: 13
180342024-01-24T19:13:21.303Z29/30 Read block 318 len 40960
180352024-01-24T19:13:21.308Z30/30 Write block 458 len 20480 data: 12 8 8 11 12
180362024-01-24T19:13:21.308Z----------------------------------------------------------------
180372024-01-24T19:13:21.308Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
180382024-01-24T19:13:21.308ZStates: Active Active Active
180392024-01-24T19:13:21.308ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
180402024-01-24T19:13:21.308Z 1 Acked 1000 Write 8 Done Done Done false
180412024-01-24T19:13:21.308Z 2 Acked 1001 Write 4 Done Done Done false
180422024-01-24T19:13:21.308Z 3 Acked 1002 Read 6 Done Done Done false
180432024-01-24T19:13:21.308Z 4 Acked 1003 Write 7 Done Done Done false
180442024-01-24T19:13:21.308Z 5 Acked 1004 Read 4 Done Done Sent false
180452024-01-24T19:13:21.308Z 6 Acked 1005 Write 1 Done Done Sent false
180462024-01-24T19:13:21.308Z 7 Acked 1006 Read 5 Done Done Sent false
180472024-01-24T19:13:21.308Z 8 Acked 1007 Read 3 Done Done Sent false
180482024-01-24T19:13:21.308Z 9 Acked 1008 Read 1 Done Done Sent false
180492024-01-24T19:13:21.308Z 10 Acked 1009 Write 7 Done Done Sent false
180502024-01-24T19:13:21.308Z 11 Acked 1010 Write 2 Done Done Sent false
180512024-01-24T19:13:21.308Z 12 Acked 1011 Write 3 Done Done Sent false
180522024-01-24T19:13:21.308Z 13 Acked 1012 Write 9 Done Done Sent false
180532024-01-24T19:13:21.309Z 14 Acked 1013 Write 8 Done Done Sent false
180542024-01-24T19:13:21.309Z 15 Acked 1014 Flush 0 Done Done Sent false
180552024-01-24T19:13:21.309Z 16 Acked 1015 Write 2 Done Done Sent false
180562024-01-24T19:13:21.309Z 17 Acked 1016 Write 5 Done Done Sent false
180572024-01-24T19:13:21.309Z 18 Acked 1017 Read 9 Done Done Sent false
180582024-01-24T19:13:21.309Z 19 Acked 1018 Read 9 Done Done Sent false
180592024-01-24T19:13:21.309Z 20 Acked 1019 Flush 0 Done Done Sent false
180602024-01-24T19:13:21.309Z 21 Acked 1020 Write 1 Done Done Sent false
180612024-01-24T19:13:21.309Z 22 Acked 1021 Write 8 Done Done Sent false
180622024-01-24T19:13:21.309Z 23 Acked 1022 Write 1 Done Done Sent false
180632024-01-24T19:13:21.309Z 24 Acked 1023 Write 2 Done Done Sent false
180642024-01-24T19:13:21.309Z 25 Acked 1024 Flush 0 Done Done Sent false
180652024-01-24T19:13:21.309Z 26 Acked 1025 Flush 0 Done Done Sent false
180662024-01-24T19:13:21.309Z 27 Acked 1026 Write 7 Done Done Sent false
180672024-01-24T19:13:21.309Z 28 Acked 1027 Write 1 Done Done Sent false
180682024-01-24T19:13:21.309Z 29 Acked 1028 Read 10 Done Done Sent false
180692024-01-24T19:13:21.309Z 30 Acked 1029 Write 5 Sent Sent Sent false
180702024-01-24T19:13:21.309Z STATES DS:0 DS:1 DS:2 TOTAL
180712024-01-24T19:13:21.309Z New 0 0 0 0
180722024-01-24T19:13:21.309Z Sent 1 1 26 28
180732024-01-24T19:13:21.309Z Done 29 29 4 62
180742024-01-24T19:13:21.310Z Skipped 0 0 0 0
180752024-01-24T19:13:21.310Z Error 0 0 0 0
180762024-01-24T19:13:21.310ZLast Flush: 1025 1025 0
180772024-01-24T19:13:21.310ZDownstairs last five completed:
180782024-01-24T19:13:21.310ZUpstairs last five completed: 30 29 28 27 26
180792024-01-24T19:13:21.310ZWrote out file "/tmp/test_repair_verify.data"
180802024-01-24T19:13:21.314Z
180812024-01-24T19:13:21.396ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180822024-01-24T19:13:21.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180832024-01-24T19:13:21.401Z 0 000-019 168 168 165 295 295 288 F F F <---
180842024-01-24T19:13:21.402Z 1 020-039 168 168 166 295 295 290 F F F <---
180852024-01-24T19:13:21.402Z 2 040-059 168 168 167 295 295 292 F F F <---
180862024-01-24T19:13:21.403Z 3 060-079 162 162 162 282 282 282 F F F
180872024-01-24T19:13:21.403Z 4 080-099 162 162 162 282 282 282 T T F <---
180882024-01-24T19:13:21.403Z 5 100-119 167 167 167 292 292 292 F F F
180892024-01-24T19:13:21.403Z 6 120-139 166 166 166 289 289 289 F F F
180902024-01-24T19:13:21.403Z 7 140-159 163 163 163 285 285 285 F F F
180912024-01-24T19:13:21.403Z 8 160-179 165 165 165 288 288 288 F F F
180922024-01-24T19:13:21.403Z 9 180-199 165 165 165 288 288 288 F F F
180932024-01-24T19:13:21.403Z 10 200-219 165 165 165 288 288 288 T T F <---
180942024-01-24T19:13:21.403Z 11 220-239 160 160 160 280 280 280 F F F
180952024-01-24T19:13:21.403Z 12 240-259 162 162 162 284 284 284 F F F
180962024-01-24T19:13:21.403Z 13 260-279 168 168 167 294 294 292 F F F <---
180972024-01-24T19:13:21.403Z 14 280-299 164 164 164 287 287 287 F F F
180982024-01-24T19:13:21.403Z 15 300-319 168 168 167 293 293 292 F F F <---
180992024-01-24T19:13:21.404Z 16 320-339 168 168 162 293 293 284 F F F <---
181002024-01-24T19:13:21.404Z 17 340-359 168 168 164 293 293 286 F F T <---
181012024-01-24T19:13:21.404Z 18 360-379 166 166 166 290 290 290 F F F
181022024-01-24T19:13:21.404Z 19 380-399 166 166 166 289 289 289 F F F
181032024-01-24T19:13:21.404Z 20 400-419 168 168 163 294 294 285 F F F <---
181042024-01-24T19:13:21.404Z 21 420-439 168 168 159 295 295 279 F F F <---
181052024-01-24T19:13:21.404Z 22 440-459 167 167 167 292 292 292 T T F <---
181062024-01-24T19:13:21.404Z 23 460-479 158 158 158 273 273 273 T T F <---
181072024-01-24T19:13:21.404Z 24 480-499 152 152 152 263 263 263 F F F
181082024-01-24T19:13:21.404Z 25 500-519 168 168 163 295 295 285 F F T <---
181092024-01-24T19:13:21.404Z 26 520-539 168 168 164 293 293 286 F F T <---
181102024-01-24T19:13:21.405Z 27 540-559 168 168 162 293 293 284 F F F <---
181112024-01-24T19:13:21.405Z 28 560-579 166 166 166 290 290 290 F F F
181122024-01-24T19:13:21.405Z 29 580-599 168 168 161 293 293 281 F F F <---
181132024-01-24T19:13:21.405ZMax gen: 168, Max flush: 295
181142024-01-24T19:13:21.405ZError: Difference in extent metadata found!
181152024-01-24T19:13:21.405ZA Difference in extent metadata is expected here
181162024-01-24T19:13:21.405ZOn loop 84
181172024-01-24T19:13:21.405Z
181182024-01-24T19:13:21.405ZVerifying data now
181192024-01-24T19:13:27.790ZNo change after verify
181202024-01-24T19:13:27.790ZLoop: 84 Downstairs dump after verify (and repair):
181212024-01-24T19:13:27.811ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181222024-01-24T19:13:27.812Z 0 000-019 168 168 168 295 295 295 F F F
181232024-01-24T19:13:27.818Z 1 020-039 168 168 168 295 295 295 F F F
181242024-01-24T19:13:27.818Z 2 040-059 168 168 168 295 295 295 F F F
181252024-01-24T19:13:27.818Z 3 060-079 162 162 162 282 282 282 F F F
181262024-01-24T19:13:27.819Z 4 080-099 169 169 169 296 296 296 F F F
181272024-01-24T19:13:27.819Z 5 100-119 167 167 167 292 292 292 F F F
181282024-01-24T19:13:27.819Z 6 120-139 166 166 166 289 289 289 F F F
181292024-01-24T19:13:27.819Z 7 140-159 163 163 163 285 285 285 F F F
181302024-01-24T19:13:27.819Z 8 160-179 165 165 165 288 288 288 F F F
181312024-01-24T19:13:27.820Z 9 180-199 165 165 165 288 288 288 F F F
181322024-01-24T19:13:27.820Z 10 200-219 169 169 169 296 296 296 F F F
181332024-01-24T19:13:27.820Z 11 220-239 160 160 160 280 280 280 F F F
181342024-01-24T19:13:27.820Z 12 240-259 162 162 162 284 284 284 F F F
181352024-01-24T19:13:27.820Z 13 260-279 168 168 168 294 294 294 F F F
181362024-01-24T19:13:27.821Z 14 280-299 164 164 164 287 287 287 F F F
181372024-01-24T19:13:27.821Z 15 300-319 168 168 168 293 293 293 F F F
181382024-01-24T19:13:27.821Z 16 320-339 168 168 168 293 293 293 F F F
181392024-01-24T19:13:27.821Z 17 340-359 168 168 168 293 293 293 F F F
181402024-01-24T19:13:27.821Z 18 360-379 166 166 166 290 290 290 F F F
181412024-01-24T19:13:27.821Z 19 380-399 166 166 166 289 289 289 F F F
181422024-01-24T19:13:27.821Z 20 400-419 168 168 168 294 294 294 F F F
181432024-01-24T19:13:27.822Z 21 420-439 168 168 168 295 295 295 F F F
181442024-01-24T19:13:27.822Z 22 440-459 169 169 169 296 296 296 F F F
181452024-01-24T19:13:27.822Z 23 460-479 169 169 169 296 296 296 F F F
181462024-01-24T19:13:27.822Z 24 480-499 152 152 152 263 263 263 F F F
181472024-01-24T19:13:27.822Z 25 500-519 168 168 168 295 295 295 F F F
181482024-01-24T19:13:27.823Z 26 520-539 168 168 168 293 293 293 F F F
181492024-01-24T19:13:27.823Z 27 540-559 168 168 168 293 293 293 F F F
181502024-01-24T19:13:27.823Z 28 560-579 166 166 166 290 290 290 F F F
181512024-01-24T19:13:27.823Z 29 580-599 168 168 168 293 293 293 F F F
181522024-01-24T19:13:27.823ZMax gen: 169, Max flush: 296
181532024-01-24T19:13:27.823Z
181542024-01-24T19:13:27.824ZBegin loop 85
181552024-01-24T19:13:27.824ZDownstairs to restart: 0
181562024-01-24T19:13:28.276Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520040481Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809}
181572024-01-24T19:13:28.276Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520482258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809}
181582024-01-24T19:13:28.276Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520510279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809}
181592024-01-24T19:13:28.276Z{"msg":"Crucible b3afaf2e-abda-4b5c-90bf-3e54e41f586b has session id: 93488834-d9b5-4bb6-a437-b2ad0b84358a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520533056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181602024-01-24T19:13:28.276Z{"msg":"Upstairs opts: Upstairs UUID: b3afaf2e-abda-4b5c-90bf-3e54e41f586b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520594555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181612024-01-24T19:13:28.276Z{"msg":"Crucible stats registered with UUID: b3afaf2e-abda-4b5c-90bf-3e54e41f586b","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520621616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181622024-01-24T19:13:28.276ZCrucible runtime is spawned
181632024-01-24T19:13:28.276Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.520986937Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809}
181642024-01-24T19:13:28.276Z{"msg":"b3afaf2e-abda-4b5c-90bf-3e54e41f586b active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521173515Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181652024-01-24T19:13:28.276Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521258292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181662024-01-24T19:13:28.276Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521295327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181672024-01-24T19:13:28.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521329231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181682024-01-24T19:13:28.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521399968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181692024-01-24T19:13:28.277Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521446693Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181702024-01-24T19:13:28.277Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.52152257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181712024-01-24T19:13:28.277Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521596772Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181722024-01-24T19:13:28.277Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521788205Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181732024-01-24T19:13:28.277Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521938977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181742024-01-24T19:13:28.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.521988224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181752024-01-24T19:13:28.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.52205165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181762024-01-24T19:13:28.277Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.522192741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181772024-01-24T19:13:28.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.523258629Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181782024-01-24T19:13:28.277Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.523397244Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181792024-01-24T19:13:28.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.52346292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181802024-01-24T19:13:28.277Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.523531117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181812024-01-24T19:13:28.277Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.523575856Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181822024-01-24T19:13:28.277Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.523650599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181832024-01-24T19:13:28.277Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.524274681Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181842024-01-24T19:13:28.277Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.524522398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181852024-01-24T19:13:28.278Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.524579328Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181862024-01-24T19:13:28.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525046374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181872024-01-24T19:13:28.279Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525140536Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181882024-01-24T19:13:28.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525239943Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181892024-01-24T19:13:28.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525405206Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181902024-01-24T19:13:28.279Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525489009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181912024-01-24T19:13:28.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525519493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181922024-01-24T19:13:28.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525579118Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181932024-01-24T19:13:28.279Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525614127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181942024-01-24T19:13:28.279Z{"msg":"[0]R flush_numbers[0..12]: [295, 295, 295, 282, 296, 292, 289, 285, 288, 288, 296, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525674154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181952024-01-24T19:13:28.279Z{"msg":"[0]R generation[0..12]: [168, 168, 168, 162, 169, 167, 166, 163, 165, 165, 169, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.52573666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181962024-01-24T19:13:28.279Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525802131Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181972024-01-24T19:13:28.279Z{"msg":"[1]R flush_numbers[0..12]: [295, 295, 295, 282, 296, 292, 289, 285, 288, 288, 296, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525835235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
181982024-01-24T19:13:28.279Zsession_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
181992024-01-24T19:13:28.279Z{"msg":"[1]R generation[0..12]: [168, 168, 168, 162, 169, 167, 166, 163, 165, 165, 169, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.525942702Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182002024-01-24T19:13:28.279Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526001881Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182012024-01-24T19:13:28.279Z{"msg":"[2]R flush_numbers[0..12]: [295, 295, 295, 282, 296, 292, 289, 285, 288, 288, 296, 280]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526065454Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182022024-01-24T19:13:28.279Z{"msg":"[2]R generation[0..12]: [168, 168, 168, 162, 169, 167, 166, 163, 165, 165, 169, 160]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526130793Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182032024-01-24T19:13:28.280Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526160927Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182042024-01-24T19:13:28.280Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.52622564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182052024-01-24T19:13:28.280Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526253935Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182062024-01-24T19:13:28.280Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526323319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182072024-01-24T19:13:28.280Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526351002Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"":"downstairs","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182082024-01-24T19:13:28.280Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526411432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182092024-01-24T19:13:28.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526437109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"0","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182102024-01-24T19:13:28.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526507703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"1","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182112024-01-24T19:13:28.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526535997Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"client":"2","":"downstairs","session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182122024-01-24T19:13:28.280Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526596829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182132024-01-24T19:13:28.280Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526625833Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182142024-01-24T19:13:28.281Z{"msg":"b3afaf2e-abda-4b5c-90bf-3e54e41f586b is now active with session: 93488834-d9b5-4bb6-a437-b2ad0b84358a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526689201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182152024-01-24T19:13:28.281Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526714093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809,"session_id":"93488834-d9b5-4bb6-a437-b2ad0b84358a"}
182162024-01-24T19:13:28.281Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:26.526741529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1809}
182172024-01-24T19:13:28.281ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182182024-01-24T19:13:28.281ZLoading write count information from file "/tmp/test_repair_verify.data"
182192024-01-24T19:13:28.281ZRun Repair workload
182202024-01-24T19:13:28.281Z01/30 Write block 243 len 24576 data: 16 16 16 18 18 16
182212024-01-24T19:13:28.281Z02/30 Write block 314 len 20480 data: 16 16 16 15 14
182222024-01-24T19:13:28.281Z03/30 Write block 545 len 12288 data: 14 16 17
182232024-01-24T19:13:28.281Z04/30 Read block 351 len 8192
182242024-01-24T19:13:28.281Z05/30 Write block 275 len 20480 data: 13 11 14 14 17
182252024-01-24T19:13:28.281Z06/30 Write block 227 len 12288 data: 10 10 9
182262024-01-24T19:13:28.281Z07/30 Write block 507 len 4096 data: 13
182272024-01-24T19:13:28.282Z08/30 Read block 106 len 32768
182282024-01-24T19:13:28.282Z09/30 Read block 59 len 40960
182292024-01-24T19:13:28.282Z10/30 Read block 60 len 36864
182302024-01-24T19:13:28.282Z11/30 Read block 549 len 8192
182312024-01-24T19:13:28.282Z12/30 Read block 29 len 40960
182322024-01-24T19:13:28.282Z13/30 Read block 390 len 12288
182332024-01-24T19:13:28.282Z14/30 Read block 130 len 24576
182342024-01-24T19:13:28.283Z15/30 Read block 419 len 12288
182352024-01-24T19:13:28.283Z16/30 Write block 47 len 16384 data: 13 15 16 18
182362024-01-24T19:13:28.283Z17/30 Read block 416 len 32768
182372024-01-24T19:13:28.283Z18/30 Read block 158 len 12288
182382024-01-24T19:13:28.283Z19/30 Flush
182392024-01-24T19:13:28.283Z20/30 Write block 559 len 28672 data: 14 11 15 12 11 10 12
182402024-01-24T19:13:28.283Z21/30 Read block 39 len 4096
182412024-01-24T19:13:28.283Z22/30 Read block 189 len 32768
182422024-01-24T19:13:28.283Z23/30 Write block 303 len 36864 data: 8 5 7 7 8 7 8 10 10
182432024-01-24T19:13:28.283Z24/30 Read block 435 len 24576
182442024-01-24T19:13:28.283Z25/30 Read block 331 len 28672
182452024-01-24T19:13:28.283Z26/30 Write block 362 len 24576 data: 15 16 17 16 16 17
182462024-01-24T19:13:28.283Z27/30 Write block 442 len 8192 data: 7 7
182472024-01-24T19:13:28.283Z28/30 Read block 284 len 24576
182482024-01-24T19:13:28.283Z29/30 Read block 140 len 24576
182492024-01-24T19:13:28.283Z30/30 Write block 422 len 40960 data: 13 13 13 13 11 15 15 17 17 16
182502024-01-24T19:13:28.283Z----------------------------------------------------------------
182512024-01-24T19:13:28.283Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
182522024-01-24T19:13:28.283ZStates: Active Active Active
182532024-01-24T19:13:28.283ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182542024-01-24T19:13:28.283Z 1 Acked 1000 Write 6 Done Done Done false
182552024-01-24T19:13:28.284Z 2 Acked 1001 Write 5 Done Done Done false
182562024-01-24T19:13:28.289Z 3 Acked 1002 Write 3 Sent Done Done false
182572024-01-24T19:13:28.289Z 4 Acked 1003 Read 2 Sent Done Done false
182582024-01-24T19:13:28.289Z 5 Acked 1004 Write 5 Sent Done Done false
182592024-01-24T19:13:28.289Z 6 Acked 1005 Write 3 Sent Done Done false
182602024-01-24T19:13:28.289Z 7 Acked 1006 Write 1 Sent Done Done false
182612024-01-24T19:13:28.289Z 8 Acked 1007 Read 8 Sent Done Done false
182622024-01-24T19:13:28.290Z 9 Acked 1008 Read 10 Sent Done Done false
182632024-01-24T19:13:28.290Z 10 Acked 1009 Read 9 Sent Done Done false
182642024-01-24T19:13:28.290Z 11 Acked 1010 Read 2 Sent Done Done false
182652024-01-24T19:13:28.290Z 12 Acked 1011 Read 10 Sent Done Done false
182662024-01-24T19:13:28.290Z 13 Acked 1012 Read 3 Sent Done Done false
182672024-01-24T19:13:28.290Z 14 Acked 1013 Read 6 Sent Done Done false
182682024-01-24T19:13:28.290Z 15 Acked 1014 Read 3 Sent Done Done false
182692024-01-24T19:13:28.290Z 16 Acked 1015 Write 4 Sent Done Done false
182702024-01-24T19:13:28.290Z 17 Acked 1016 Read 8 Sent Done Done false
182712024-01-24T19:13:28.290Z 18 Acked 1017 Read 3 Sent Done Done false
182722024-01-24T19:13:28.290Z 19 Acked 1018 Flush 0 Sent Done Done false
182732024-01-24T19:13:28.290Z 20 Acked 1019 Write 7 Sent Done Done false
182742024-01-24T19:13:28.290Z 21 Acked 1020 Read 1 Sent Done Done false
182752024-01-24T19:13:28.290Z 22 Acked 1021 Read 8 Sent Done Done false
182762024-01-24T19:13:28.290Z 23 Acked 1022 Write 9 Sent Done Done false
182772024-01-24T19:13:28.290Z 24 Acked 1023 Read 6 Sent Done Done false
182782024-01-24T19:13:28.291Z 25 Acked 1024 Read 7 Sent Done Done false
182792024-01-24T19:13:28.291Z 26 Acked 1025 Write 6 Sent Done Done false
182802024-01-24T19:13:28.291Z 27 Acked 1026 Write 2 Sent Done Done false
182812024-01-24T19:13:28.291Z 28 Acked 1027 Read 6 Sent Done Done false
182822024-01-24T19:13:28.291Z 29 Acked 1028 Read 6 Sent Done Done false
182832024-01-24T19:13:28.291Z 30 Acked 1029 Write 10 Sent Sent Sent false
182842024-01-24T19:13:28.291Z STATES DS:0 DS:1 DS:2 TOTAL
182852024-01-24T19:13:28.291Z New 0 0 0 0
182862024-01-24T19:13:28.291Z Sent 28 1 1 30
182872024-01-24T19:13:28.291Z Done 2 29 29 60
182882024-01-24T19:13:28.291Z Skipped 0 0 0 0
182892024-01-24T19:13:28.291Z Error 0 0 0 0
182902024-01-24T19:13:28.291ZLast Flush: 0 1018 1018
182912024-01-24T19:13:28.291ZDownstairs last five completed:
182922024-01-24T19:13:28.291ZUpstairs last five completed: 30 29 28 27 26
182932024-01-24T19:13:28.291ZWrote out file "/tmp/test_repair_verify.data"
182942024-01-24T19:13:28.291Z
182952024-01-24T19:13:28.291ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182962024-01-24T19:13:28.292ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182972024-01-24T19:13:28.292Z 0 000-019 168 168 168 295 295 295 F F F
182982024-01-24T19:13:28.292Z 1 020-039 168 168 168 295 295 295 F F F
182992024-01-24T19:13:28.292Z 2 040-059 168 170 170 295 297 297 F F F <---
183002024-01-24T19:13:28.292Z 3 060-079 162 162 162 282 282 282 F F F
183012024-01-24T19:13:28.292Z 4 080-099 169 169 169 296 296 296 F F F
183022024-01-24T19:13:28.292Z 5 100-119 167 167 167 292 292 292 F F F
183032024-01-24T19:13:28.292Z 6 120-139 166 166 166 289 289 289 F F F
183042024-01-24T19:13:28.292Z 7 140-159 163 163 163 285 285 285 F F F
183052024-01-24T19:13:28.292Z 8 160-179 165 165 165 288 288 288 F F F
183062024-01-24T19:13:28.293Z 9 180-199 165 165 165 288 288 288 F F F
183072024-01-24T19:13:28.293Z 10 200-219 169 169 169 296 296 296 F F F
183082024-01-24T19:13:28.293Z 11 220-239 160 170 170 280 297 297 F F F <---
183092024-01-24T19:13:28.293Z 12 240-259 162 170 170 284 297 297 T F F <---
183102024-01-24T19:13:28.293Z 13 260-279 168 170 170 294 297 297 F F F <---
183112024-01-24T19:13:28.293Z 14 280-299 164 164 164 287 287 287 F F F
183122024-01-24T19:13:28.293Z 15 300-319 168 170 170 293 297 297 T T T <---
183132024-01-24T19:13:28.293Z 16 320-339 168 168 168 293 293 293 F F F
183142024-01-24T19:13:28.293Z 17 340-359 168 168 168 293 293 293 F F F
183152024-01-24T19:13:28.293Z 18 360-379 166 166 166 290 290 290 F T T <---
183162024-01-24T19:13:28.293Z 19 380-399 166 166 166 289 289 289 F F F
183172024-01-24T19:13:28.293Z 20 400-419 168 168 168 294 294 294 F F F
183182024-01-24T19:13:28.293Z 21 420-439 168 168 168 295 295 295 F T T <---
183192024-01-24T19:13:28.293Z 22 440-459 169 169 169 296 296 296 F T T <---
183202024-01-24T19:13:28.293Z 23 460-479 169 169 169 296 296 296 F F F
183212024-01-24T19:13:28.293Z 24 480-499 152 152 152 263 263 263 F F F
183222024-01-24T19:13:28.293Z 25 500-519 168 170 170 295 297 297 F F F <---
183232024-01-24T19:13:28.293Z 26 520-539 168 168 168 293 293 293 F F F
183242024-01-24T19:13:28.293Z 27 540-559 168 170 170 293 297 297 F T T <---
183252024-01-24T19:13:28.293Z 28 560-579 166 166 166 290 290 290 F T T <---
183262024-01-24T19:13:28.293Z 29 580-599 168 168 168 293 293 293 F F F
183272024-01-24T19:13:28.293ZMax gen: 170, Max flush: 297
183282024-01-24T19:13:28.293ZError: Difference in extent metadata found!
183292024-01-24T19:13:28.294ZA Difference in extent metadata is expected here
183302024-01-24T19:13:28.294ZOn loop 85
183312024-01-24T19:13:28.294Z
183322024-01-24T19:13:28.294ZVerifying data now
183332024-01-24T19:13:33.957ZNo change after verify
183342024-01-24T19:13:33.957ZLoop: 85 Downstairs dump after verify (and repair):
183352024-01-24T19:13:33.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183362024-01-24T19:13:33.979Z 0 000-019 168 168 168 295 295 295 F F F
183372024-01-24T19:13:33.979Z 1 020-039 168 168 168 295 295 295 F F F
183382024-01-24T19:13:33.979Z 2 040-059 170 170 170 297 297 297 F F F
183392024-01-24T19:13:33.980Z 3 060-079 162 162 162 282 282 282 F F F
183402024-01-24T19:13:33.980Z 4 080-099 169 169 169 296 296 296 F F F
183412024-01-24T19:13:33.980Z 5 100-119 167 167 167 292 292 292 F F F
183422024-01-24T19:13:33.980Z 6 120-139 166 166 166 289 289 289 F F F
183432024-01-24T19:13:33.980Z 7 140-159 163 163 163 285 285 285 F F F
183442024-01-24T19:13:33.980Z 8 160-179 165 165 165 288 288 288 F F F
183452024-01-24T19:13:33.980Z 9 180-199 165 165 165 288 288 288 F F F
183462024-01-24T19:13:33.980Z 10 200-219 169 169 169 296 296 296 F F F
183472024-01-24T19:13:33.980Z 11 220-239 170 170 170 297 297 297 F F F
183482024-01-24T19:13:33.980Z 12 240-259 170 170 170 297 297 297 F F F
183492024-01-24T19:13:33.980Z 13 260-279 170 170 170 297 297 297 F F F
183502024-01-24T19:13:33.980Z 14 280-299 164 164 164 287 287 287 F F F
183512024-01-24T19:13:33.980Z 15 300-319 171 171 171 298 298 298 F F F
183522024-01-24T19:13:33.987Z 16 320-339 168 168 168 293 293 293 F F F
183532024-01-24T19:13:33.987Z 17 340-359 168 168 168 293 293 293 F F F
183542024-01-24T19:13:33.987Z 18 360-379 171 171 171 298 298 298 F F F
183552024-01-24T19:13:33.987Z 19 380-399 166 166 166 289 289 289 F F F
183562024-01-24T19:13:33.987Z 20 400-419 168 168 168 294 294 294 F F F
183572024-01-24T19:13:33.987Z 21 420-439 171 171 171 298 298 298 F F F
183582024-01-24T19:13:33.987Z 22 440-459 171 171 171 298 298 298 F F F
183592024-01-24T19:13:33.988Z 23 460-479 169 169 169 296 296 296 F F F
183602024-01-24T19:13:33.988Z 24 480-499 152 152 152 263 263 263 F F F
183612024-01-24T19:13:33.988Z 25 500-519 170 170 170 297 297 297 F F F
183622024-01-24T19:13:33.988Z 26 520-539 168 168 168 293 293 293 F F F
183632024-01-24T19:13:33.988Z 27 540-559 171 171 171 298 298 298 F F F
183642024-01-24T19:13:33.988Z 28 560-579 171 171 171 298 298 298 F F F
183652024-01-24T19:13:33.988Z 29 580-599 168 168 168 293 293 293 F F F
183662024-01-24T19:13:33.988ZMax gen: 171, Max flush: 298
183672024-01-24T19:13:33.988Z
183682024-01-24T19:13:33.988ZBegin loop 86
183692024-01-24T19:13:33.988ZDownstairs to restart: 2
183702024-01-24T19:13:34.014Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.691439203Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817}
183712024-01-24T19:13:34.014Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.691914487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817}
183722024-01-24T19:13:34.019Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.691987868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817}
183732024-01-24T19:13:34.019Z{Crucible runtime is spawned
183742024-01-24T19:13:34.019Z"msg":"Crucible fffdb8ad-2197-4a9f-aa05-21281ee47403 has session id: d07b52c0-54fe-4c83-a28e-bb2141d13f38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692243579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183752024-01-24T19:13:34.019Z{"msg":"Upstairs opts: Upstairs UUID: fffdb8ad-2197-4a9f-aa05-21281ee47403, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692349915Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183762024-01-24T19:13:34.020Z{"msg":"Crucible stats registered with UUID: fffdb8ad-2197-4a9f-aa05-21281ee47403","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692424781Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183772024-01-24T19:13:34.020Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692499692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817}
183782024-01-24T19:13:34.020Z{"msg":"fffdb8ad-2197-4a9f-aa05-21281ee47403 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692638087Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183792024-01-24T19:13:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692717181Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183802024-01-24T19:13:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69279862Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183812024-01-24T19:13:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.692965745Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183822024-01-24T19:13:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693100778Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183832024-01-24T19:13:34.020Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693189691Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183842024-01-24T19:13:34.020Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693288985Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183852024-01-24T19:13:34.021Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693373083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183862024-01-24T19:13:34.021Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693520026Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183872024-01-24T19:13:34.021Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693732612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183882024-01-24T19:13:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69383331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183892024-01-24T19:13:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693863479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183902024-01-24T19:13:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.693903511Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183912024-01-24T19:13:34.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.694678042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183922024-01-24T19:13:34.021Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.694797359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183932024-01-24T19:13:34.021Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.695035445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183942024-01-24T19:13:34.022Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.695095847Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183952024-01-24T19:13:34.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.695189989Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183962024-01-24T19:13:34.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69524737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183972024-01-24T19:13:34.027Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.695752954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183982024-01-24T19:13:34.028Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.696401463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
183992024-01-24T19:13:34.028Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.696515995Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184002024-01-24T19:13:34.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69672225Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184012024-01-24T19:13:34.028Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.696811446Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184022024-01-24T19:13:34.028Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.696857869Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184032024-01-24T19:13:34.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697489373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184042024-01-24T19:13:34.030Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69757696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184052024-01-24T19:13:34.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697706415Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184062024-01-24T19:13:34.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697732519Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184072024-01-24T19:13:34.030Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697776726Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184082024-01-24T19:13:34.030Z{"msg":"[0]R flush_numbers[0..12]: [295, 295, 297, 282, 296, 292, 289, 285, 288, 288, 296, 297]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697847174Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184092024-01-24T19:13:34.030Z{"msg":"[0]R generation[0..12]: [168, 168, 170, 162, 169, 167, 166, 163, 165, 165, 169, 170]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697875073Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184102024-01-24T19:13:34.030Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697899505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184112024-01-24T19:13:34.030Z{"msg":"[1]R flush_numbers[0..12]: [295, 295, 297, 282, 296, 292, 289, 285, 288, 288, 296, 297]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69794379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184122024-01-24T19:13:34.030Z{"msg":"[1]R generation[0..12]: [168, 168, 170, 162, 169, 167, 166, 163, 165, 165, 169, 170]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697970706Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184132024-01-24T19:13:34.031Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.697994557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184142024-01-24T19:13:34.031ZWait for a query_work_queue command to finish before sending IO
184152024-01-24T19:13:34.031Z{"msg":"[2]R flush_numbers[0..12]: [295, 295, 297, 282, 296, 292, 289, 285, 288, 288, 296, 297]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698045374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184162024-01-24T19:13:34.031Z{"msg":"[2]R generation[0..12]: [168, 168, 170, 162, 169, 167, 166, 163, 165, 165, 169, 170]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698072692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184172024-01-24T19:13:34.031Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698097074Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184182024-01-24T19:13:34.031Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698139842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184192024-01-24T19:13:34.031Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698165839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184202024-01-24T19:13:34.031Z{"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698187523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184212024-01-24T19:13:34.031Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698229858Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"":"downstairs","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184222024-01-24T19:13:34.031Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698255464Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184232024-01-24T19:13:34.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69827594Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"0","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184242024-01-24T19:13:34.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698318147Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"1","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184252024-01-24T19:13:34.033Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698343688Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"client":"2","":"downstairs","session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184262024-01-24T19:13:34.033Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698365768Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184272024-01-24T19:13:34.033Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698406892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184282024-01-24T19:13:34.033Z{"msg":"fffdb8ad-2197-4a9f-aa05-21281ee47403 is now active with session: d07b52c0-54fe-4c83-a28e-bb2141d13f38","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.69843194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184292024-01-24T19:13:34.033Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698452066Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817,"session_id":"d07b52c0-54fe-4c83-a28e-bb2141d13f38"}
184302024-01-24T19:13:34.033Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:32.698492931Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1817}
184312024-01-24T19:13:34.033ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184322024-01-24T19:13:34.033ZLoading write count information from file "/tmp/test_repair_verify.data"
184332024-01-24T19:13:34.033ZRun Repair workload
184342024-01-24T19:13:34.033Z01/30 Write block 259 len 8192 data: 11 12
184352024-01-24T19:13:34.033Z02/30 Read block 500 len 28672
184362024-01-24T19:13:34.038Z03/30 Write block 130 len 32768 data: 13 14 14 12 16 16 17 16
184372024-01-24T19:13:34.038Z04/30 Read block 559 len 8192
184382024-01-24T19:13:34.038Z05/30 Read block 24 len 8192
184392024-01-24T19:13:34.038Z06/30 Write block 210 len 20480 data: 14 12 12 11 8
184402024-01-24T19:13:34.038Z07/30 Read block 467 len 24576
184412024-01-24T19:13:34.038Z08/30 Flush
184422024-01-24T19:13:34.038Z09/30 Write block 69 len 16384 data: 6 7 7 8
184432024-01-24T19:13:34.038Z10/30 Read block 168 len 4096
184442024-01-24T19:13:34.041Z11/30 Write block 74 len 32768 data: 7 7 8 10 12 12 12 14
184452024-01-24T19:13:34.041Z12/30 Write block 567 len 20480 data: 18 19 20 14 14
184462024-01-24T19:13:34.041Z13/30 Write block 399 len 24576 data: 12 13 14 13 8 7
184472024-01-24T19:13:34.044Z14/30 Flush
184482024-01-24T19:13:34.048Z15/30 Write block 287 len 36864 data: 14 15 14 12 10 10 11 15 13
184492024-01-24T19:13:34.051Z16/30 Write block 238 len 32768 data: 11 11 10 10 12 17 17 17
184502024-01-24T19:13:34.051Z17/30 Write block 479 len 16384 data: 9 12 13 13
184512024-01-24T19:13:34.052Z18/30 Read block 75 len 28672
184522024-01-24T19:13:34.055Z19/30 Write block 75 len 24576 data: 8 9 11 13 13 13
184532024-01-24T19:13:34.055Z20/30 Write block 343 len 28672 data: 14 13 13 11 13 11 13
184542024-01-24T19:13:34.056Z21/30 Write block 44 len 8192 data: 15 18
184552024-01-24T19:13:34.060Z22/30 Read block 49 len 16384
184562024-01-24T19:13:34.060Z23/30 Write block 253 len 4096 data: 7
184572024-01-24T19:13:34.060Z24/30 Write block 535 len 24576 data: 12 11 15 14 14 12
184582024-01-24T19:13:34.060Z25/30 Write block 232 len 36864 data: 12 14 15 12 14 13 12 12 11
184592024-01-24T19:13:34.064Z26/30 Read block 310 len 32768
184602024-01-24T19:13:34.064Z27/30 Read block 325 len 28672
184612024-01-24T19:13:34.068Z28/30 Read block 34 len 32768
184622024-01-24T19:13:34.068Z29/30 Read block 438 len 20480
184632024-01-24T19:13:34.072Z30/30 Write block 371 len 32768 data: 14 12 13 12 16 14 14 10
184642024-01-24T19:13:34.072Z----------------------------------------------------------------
184652024-01-24T19:13:34.072Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
184662024-01-24T19:13:34.072ZStates: Active Active Active
184672024-01-24T19:13:34.072ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184682024-01-24T19:13:34.072Z 1 Acked 1000 Write 2 Done Done Sent false
184692024-01-24T19:13:34.072Z 2 Acked 1001 Read 7 Done Done Sent false
184702024-01-24T19:13:34.073Z 3 Acked 1002 Write 8 Done Done Sent false
184712024-01-24T19:13:34.073Z 4 Acked 1003 Read 2 Done Done Sent false
184722024-01-24T19:13:34.073Z 5 Acked 1004 Read 2 Done Done Sent false
184732024-01-24T19:13:34.073Z 6 Acked 1005 Write 5 Done Done Sent false
184742024-01-24T19:13:34.073Z 7 Acked 1006 Read 6 Done Done Sent false
184752024-01-24T19:13:34.073Z 8 Acked 1007 Flush 0 Done Done Sent false
184762024-01-24T19:13:34.073Z 9 Acked 1008 Write 4 Done Done Sent false
184772024-01-24T19:13:34.073Z 10 Acked 1009 Read 1 Done Done Sent false
184782024-01-24T19:13:34.073Z 11 Acked 1010 Write 8 Done Done Sent false
184792024-01-24T19:13:34.073Z 12 Acked 1011 Write 5 Done Done Sent false
184802024-01-24T19:13:34.073Z 13 Acked 1012 Write 6 Done Done Sent false
184812024-01-24T19:13:34.073Z 14 Acked 1013 Flush 0 Done Done Sent false
184822024-01-24T19:13:34.074Z 15 Acked 1014 Write 9 Done Done Sent false
184832024-01-24T19:13:34.074Z 16 Acked 1015 Write 8 Done Done Sent false
184842024-01-24T19:13:34.074Z 17 Acked 1016 Write 4 Done Done Sent false
184852024-01-24T19:13:34.074Z 18 Acked 1017 Read 7 Done Done Sent false
184862024-01-24T19:13:34.075Z 19 Acked 1018 Write 6 Done Done Sent false
184872024-01-24T19:13:34.075Z 20 Acked 1019 Write 7 Done Done Sent false
184882024-01-24T19:13:34.075Z 21 Acked 1020 Write 2 Done Done Sent false
184892024-01-24T19:13:34.075Z 22 Acked 1021 Read 4 Done Done Sent false
184902024-01-24T19:13:34.075Z 23 Acked 1022 Write 1 Done Done Sent false
184912024-01-24T19:13:34.075Z 24 Acked 1023 Write 6 Done Done Sent false
184922024-01-24T19:13:34.075Z 25 Acked 1024 Write 9 Done Done Sent false
184932024-01-24T19:13:34.075Z 26 Acked 1025 Read 8 Done Done Sent false
184942024-01-24T19:13:34.075Z 27 Acked 1026 Read 7 Done Done Sent false
184952024-01-24T19:13:34.075Z 28 Acked 1027 Read 8 Done Done Sent false
184962024-01-24T19:13:34.075Z 29 Acked 1028 Read 5 Done Done Sent false
184972024-01-24T19:13:34.075Z 30 Acked 1029 Write 8 Sent Sent Sent false
184982024-01-24T19:13:34.076Z STATES DS:0 DS:1 DS:2 TOTAL
184992024-01-24T19:13:34.076Z New 0 0 0 0
185002024-01-24T19:13:34.076Z Sent 1 1 30 32
185012024-01-24T19:13:34.076Z Done 29 29 0 58
185022024-01-24T19:13:34.076Z Skipped 0 0 0 0
185032024-01-24T19:13:34.076Z Error 0 0 0 0
185042024-01-24T19:13:34.076ZLast Flush: 1013 1013 0
185052024-01-24T19:13:34.076ZDownstairs last five completed:
185062024-01-24T19:13:34.076ZUpstairs last five completed: 30 29 28 27 26
185072024-01-24T19:13:34.076ZWrote out file "/tmp/test_repair_verify.data"
185082024-01-24T19:13:34.080Z
185092024-01-24T19:13:34.084ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185102024-01-24T19:13:34.114ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185112024-01-24T19:13:34.114Z 0 000-019 168 168 168 295 295 295 F F F
185122024-01-24T19:13:34.114Z 1 020-039 168 168 168 295 295 295 F F F
185132024-01-24T19:13:34.117Z 2 040-059 170 170 170 297 297 297 T T F <---
185142024-01-24T19:13:34.118Z 3 060-079 172 172 162 300 300 282 T T F <---
185152024-01-24T19:13:34.118Z 4 080-099 172 172 169 300 300 296 T T F <---
185162024-01-24T19:13:34.118Z 5 100-119 167 167 167 292 292 292 F F F
185172024-01-24T19:13:34.118Z 6 120-139 172 172 166 299 299 289 F F F <---
185182024-01-24T19:13:34.118Z 7 140-159 163 163 163 285 285 285 F F F
185192024-01-24T19:13:34.118Z 8 160-179 165 165 165 288 288 288 F F F
185202024-01-24T19:13:34.118Z 9 180-199 165 165 165 288 288 288 F F F
185212024-01-24T19:13:34.118Z 10 200-219 172 172 169 299 299 296 F F F <---
185222024-01-24T19:13:34.118Z 11 220-239 170 170 170 297 297 297 T T F <---
185232024-01-24T19:13:34.118Z 12 240-259 172 172 170 299 299 297 T T F <---
185242024-01-24T19:13:34.118Z 13 260-279 172 172 170 299 299 297 F F F <---
185252024-01-24T19:13:34.118Z 14 280-299 164 164 164 287 287 287 T T F <---
185262024-01-24T19:13:34.118Z 15 300-319 171 171 171 298 298 298 F F F
185272024-01-24T19:13:34.119Z 16 320-339 168 168 168 293 293 293 F F F
185282024-01-24T19:13:34.119Z 17 340-359 168 168 168 293 293 293 T T F <---
185292024-01-24T19:13:34.119Z 18 360-379 171 171 171 298 298 298 T T F <---
185302024-01-24T19:13:34.119Z 19 380-399 172 172 166 300 300 289 F F F <---
185312024-01-24T19:13:34.119Z 20 400-419 172 172 168 300 300 294 F F F <---
185322024-01-24T19:13:34.119Z 21 420-439 171 171 171 298 298 298 F F F
185332024-01-24T19:13:34.119Z 22 440-459 171 171 171 298 298 298 F F F
185342024-01-24T19:13:34.119Z 23 460-479 169 169 169 296 296 296 T T F <---
185352024-01-24T19:13:34.119Z 24 480-499 152 152 152 263 263 263 T T F <---
185362024-01-24T19:13:34.119Z 25 500-519 170 170 170 297 297 297 F F F
185372024-01-24T19:13:34.119Z 26 520-539 168 168 168 293 293 293 T T F <---
185382024-01-24T19:13:34.119Z 27 540-559 171 171 171 298 298 298 T T F <---
185392024-01-24T19:13:34.120Z 28 560-579 172 172 171 300 300 298 F F F <---
185402024-01-24T19:13:34.120Z 29 580-599 168 168 168 293 293 293 F F F
185412024-01-24T19:13:34.120ZMax gen: 172, Max flush: 300
185422024-01-24T19:13:34.120ZError: Difference in extent metadata found!
185432024-01-24T19:13:34.125ZA Difference in extent metadata is expected here
185442024-01-24T19:13:34.125ZOn loop 86
185452024-01-24T19:13:34.125Z
185462024-01-24T19:13:34.125ZVerifying data now
185472024-01-24T19:13:39.659ZNo change after verify
185482024-01-24T19:13:39.659ZLoop: 86 Downstairs dump after verify (and repair):
185492024-01-24T19:13:39.682ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185502024-01-24T19:13:39.682Z 0 000-019 168 168 168 295 295 295 F F F
185512024-01-24T19:13:39.690Z 1 020-039 168 168 168 295 295 295 F F F
185522024-01-24T19:13:39.690Z 2 040-059 173 173 173 301 301 301 F F F
185532024-01-24T19:13:39.690Z 3 060-079 173 173 173 301 301 301 F F F
185542024-01-24T19:13:39.690Z 4 080-099 173 173 173 301 301 301 F F F
185552024-01-24T19:13:39.690Z 5 100-119 167 167 167 292 292 292 F F F
185562024-01-24T19:13:39.690Z 6 120-139 172 172 172 299 299 299 F F F
185572024-01-24T19:13:39.690Z 7 140-159 163 163 163 285 285 285 F F F
185582024-01-24T19:13:39.690Z 8 160-179 165 165 165 288 288 288 F F F
185592024-01-24T19:13:39.690Z 9 180-199 165 165 165 288 288 288 F F F
185602024-01-24T19:13:39.690Z 10 200-219 172 172 172 299 299 299 F F F
185612024-01-24T19:13:39.690Z 11 220-239 173 173 173 301 301 301 F F F
185622024-01-24T19:13:39.690Z 12 240-259 173 173 173 301 301 301 F F F
185632024-01-24T19:13:39.690Z 13 260-279 172 172 172 299 299 299 F F F
185642024-01-24T19:13:39.690Z 14 280-299 173 173 173 301 301 301 F F F
185652024-01-24T19:13:39.690Z 15 300-319 171 171 171 298 298 298 F F F
185662024-01-24T19:13:39.690Z 16 320-339 168 168 168 293 293 293 F F F
185672024-01-24T19:13:39.690Z 17 340-359 173 173 173 301 301 301 F F F
185682024-01-24T19:13:39.690Z 18 360-379 173 173 173 301 301 301 F F F
185692024-01-24T19:13:39.690Z 19 380-399 172 172 172 300 300 300 F F F
185702024-01-24T19:13:39.690Z 20 400-419 172 172 172 300 300 300 F F F
185712024-01-24T19:13:39.691Z 21 420-439 171 171 171 298 298 298 F F F
185722024-01-24T19:13:39.692Z 22 440-459 171 171 171 298 298 298 F F F
185732024-01-24T19:13:39.692Z 23 460-479 173 173 173 301 301 301 F F F
185742024-01-24T19:13:39.692Z 24 480-499 173 173 173 301 301 301 F F F
185752024-01-24T19:13:39.692Z 25 500-519 170 170 170 297 297 297 F F F
185762024-01-24T19:13:39.692Z 26 520-539 173 173 173 301 301 301 F F F
185772024-01-24T19:13:39.692Z 27 540-559 173 173 173 301 301 301 F F F
185782024-01-24T19:13:39.692Z 28 560-579 172 172 172 300 300 300 F F F
185792024-01-24T19:13:39.692Z 29 580-599 168 168 168 293 293 293 F F F
185802024-01-24T19:13:39.692ZMax gen: 173, Max flush: 301
185812024-01-24T19:13:39.692Z
185822024-01-24T19:13:39.692ZBegin loop 87
185832024-01-24T19:13:39.692ZDownstairs to restart: 0
185842024-01-24T19:13:39.718Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.393213048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825}
185852024-01-24T19:13:39.745Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.393708897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825}
185862024-01-24T19:13:39.745Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39382674Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825}
185872024-01-24T19:13:39.745Z{"msg":"Crucible 1986c1c0-f5ad-4e1c-b89e-45b409351489 has session id: 0e0effd5-dbde-4d3c-a96e-c968d4e212d3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39386944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185882024-01-24T19:13:39.745Z{"msg":"Upstairs opts: Upstairs UUID: 1986c1c0-f5ad-4e1c-b89e-45b409351489, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.393931944Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185892024-01-24T19:13:39.746Z{"msg":"Crucible stats registered with UUID: 1986c1c0-f5ad-4e1c-b89e-45b409351489","v":0,"name":"crucible","level":30Crucible runtime is spawned
185902024-01-24T19:13:39.746Z,"time":"2024-01-24T19:13:38.394012265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185912024-01-24T19:13:39.746Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394193168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825}
185922024-01-24T19:13:39.746Z{"msg":"1986c1c0-f5ad-4e1c-b89e-45b409351489 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394432052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185932024-01-24T19:13:39.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394562471Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185942024-01-24T19:13:39.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394655425Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185952024-01-24T19:13:39.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39470385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185962024-01-24T19:13:39.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394748011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185972024-01-24T19:13:39.746Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.394923286Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185982024-01-24T19:13:39.746Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395090036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
185992024-01-24T19:13:39.748Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395289367Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186002024-01-24T19:13:39.748Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395359765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186012024-01-24T19:13:39.748Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395405428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186022024-01-24T19:13:39.748Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395503981Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186032024-01-24T19:13:39.748Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.395633048Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186042024-01-24T19:13:39.748Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39568025Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186052024-01-24T19:13:39.748Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.396282649Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186062024-01-24T19:13:39.749Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.396372369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186072024-01-24T19:13:39.749Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.396401058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186082024-01-24T19:13:39.749Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39650669Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186092024-01-24T19:13:39.749Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.396639968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186102024-01-24T19:13:39.749Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.396669166Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186112024-01-24T19:13:39.749Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.397455844Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186122024-01-24T19:13:39.749Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.397699568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186132024-01-24T19:13:39.749Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398036589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186142024-01-24T19:13:39.749Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398473136Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186152024-01-24T19:13:39.749Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39855826Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186162024-01-24T19:13:39.749Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398602884Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186172024-01-24T19:13:39.749Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398711803Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186182024-01-24T19:13:39.749Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39883689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186192024-01-24T19:13:39.751Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398900391Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186202024-01-24T19:13:39.751Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398931705Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186212024-01-24T19:13:39.751Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.398956693Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186222024-01-24T19:13:39.751Z{"msg":"[0]R flush_numbers[0..12]: [295, 295, 301, 301, 301, 292, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399018843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186232024-01-24T19:13:39.751Z{"msg":"[0]R generation[0..12]: [168, 168, 173, 173, 173, 167, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399069552Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186242024-01-24T19:13:39.751Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399096755Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186252024-01-24T19:13:39.751Z{"msg":"[1]R flush_numbers[0..12]: [295, 295, 301, 301, 301, 292, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399141751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186262024-01-24T19:13:39.751Z{"msg":"[1]R generation[0..12]: [168, 168, 173, 173, 173, 167, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39916817Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186272024-01-24T19:13:39.751ZWait for a query_work_queue command to finish before sending IO
186282024-01-24T19:13:39.751Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399221234Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186292024-01-24T19:13:39.752Z{"msg":"[2]R flush_numbers[0..12]: [295, 295, 301, 301, 301, 292, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399248172Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186302024-01-24T19:13:39.752Z{"msg":"[2]R generation[0..12]: [168, 168, 173, 173, 173, 167, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399298686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186312024-01-24T19:13:39.753Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399325463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186322024-01-24T19:13:39.753Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399348695Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186332024-01-24T19:13:39.753Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399403227Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186342024-01-24T19:13:39.753Z{"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399427592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186352024-01-24T19:13:39.753Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399450201Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"":"downstairs","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186362024-01-24T19:13:39.753Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.39949343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186372024-01-24T19:13:39.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399516647Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"0","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186382024-01-24T19:13:39.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399538179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"1","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186392024-01-24T19:13:39.753Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399588823Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"client":"2","":"downstairs","session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186402024-01-24T19:13:39.753Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399614094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186412024-01-24T19:13:39.753Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399656223Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186422024-01-24T19:13:39.753Z{"msg":"1986c1c0-f5ad-4e1c-b89e-45b409351489 is now active with session: 0e0effd5-dbde-4d3c-a96e-c968d4e212d3","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399680909Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186432024-01-24T19:13:39.753Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399721919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825,"session_id":"0e0effd5-dbde-4d3c-a96e-c968d4e212d3"}
186442024-01-24T19:13:39.753Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:38.399745706Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1825}
186452024-01-24T19:13:39.753ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186462024-01-24T19:13:39.753ZLoading write count information from file "/tmp/test_repair_verify.data"
186472024-01-24T19:13:39.753ZRun Repair workload
186482024-01-24T19:13:39.754Z01/30 Write block 482 len 24576 data: 14 10 10 10 11 9
186492024-01-24T19:13:39.758Z02/30 Write block 533 len 40960 data: 11 13 13 12 16 15 15 13 13 15
186502024-01-24T19:13:39.758Z03/30 Write block 365 len 40960 data: 17 17 18 16 16 15 15 13 14 13
186512024-01-24T19:13:39.758Z04/30 Read block 119 len 8192
186522024-01-24T19:13:39.758Z05/30 Flush
186532024-01-24T19:13:39.758Z06/30 Write block 64 len 24576 data: 14 11 9 9 6 7
186542024-01-24T19:13:39.758Z07/30 Write block 576 len 36864 data: 15 10 12 12 11 11 10 13 14
186552024-01-24T19:13:39.759Z08/30 Read block 392 len 36864
186562024-01-24T19:13:39.759Z09/30 Read block 84 len 8192
186572024-01-24T19:13:39.759Z10/30 Read block 305 len 32768
186582024-01-24T19:13:39.759Z11/30 Read block 313 len 20480
186592024-01-24T19:13:39.759Z12/30 Write block 372 len 24576 data: 14 15 14 17 15 15
186602024-01-24T19:13:39.759Z13/30 Write block 541 len 20480 data: 14 16 14 16 15
186612024-01-24T19:13:39.759Z14/30 Flush
186622024-01-24T19:13:39.759Z15/30 Write block 573 len 28672 data: 14 14 15 16 11 13 13
186632024-01-24T19:13:39.759Z16/30 Flush
186642024-01-24T19:13:39.759Z17/30 Write block 28 len 36864 data: 17 17 15 17 15 15 15 9 9
186652024-01-24T19:13:39.759Z18/30 Read block 409 len 32768
186662024-01-24T19:13:39.763Z19/30 Read block 26 len 28672
186672024-01-24T19:13:39.763Z20/30 Read block 109 len 24576
186682024-01-24T19:13:39.766Z21/30 Flush
186692024-01-24T19:13:39.766Z22/30 Flush
186702024-01-24T19:13:39.771Z23/30 Write block 105 len 24576 data: 16 17 18 18 19 18
186712024-01-24T19:13:39.771Z24/30 Write block 533 len 20480 data: 12 14 14 13 17
186722024-01-24T19:13:39.772Z25/30 Read block 409 len 32768
186732024-01-24T19:13:39.775Z26/30 Write block 269 len 32768 data: 12 13 13 16 17 14 14 12
186742024-01-24T19:13:39.775Z27/30 Write block 587 len 12288 data: 13 13 11
186752024-01-24T19:13:39.775Z28/30 Read block 563 len 40960
186762024-01-24T19:13:39.778Z29/30 Read block 501 len 36864
186772024-01-24T19:13:39.779Z30/30 Read block 371 len 32768
186782024-01-24T19:13:39.783Z----------------------------------------------------------------
186792024-01-24T19:13:39.783Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
186802024-01-24T19:13:39.783ZStates: Active Active Active
186812024-01-24T19:13:39.783ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
186822024-01-24T19:13:39.783Z 1 Acked 1000 Write 6 Sent Done Done false
186832024-01-24T19:13:39.783Z 2 Acked 1001 Write 10 Sent Done Done false
186842024-01-24T19:13:39.784Z 3 Acked 1002 Write 10 Sent Done Done false
186852024-01-24T19:13:39.784Z 4 Acked 1003 Read 2 Sent Done Done false
186862024-01-24T19:13:39.784Z 5 Acked 1004 Flush 0 Sent Done Done false
186872024-01-24T19:13:39.784Z 6 Acked 1005 Write 6 Sent Done Done false
186882024-01-24T19:13:39.784Z 7 Acked 1006 Write 9 Sent Done Done false
186892024-01-24T19:13:39.784Z 8 Acked 1007 Read 9 Sent Done Done false
186902024-01-24T19:13:39.784Z 9 Acked 1008 Read 2 Sent Done Done false
186912024-01-24T19:13:39.784Z 10 Acked 1009 Read 8 Sent Done Done false
186922024-01-24T19:13:39.784Z 11 Acked 1010 Read 5 Sent Done Done false
186932024-01-24T19:13:39.784Z 12 Acked 1011 Write 6 Sent Done Done false
186942024-01-24T19:13:39.784Z 13 Acked 1012 Write 5 Sent Done Done false
186952024-01-24T19:13:39.784Z 14 Acked 1013 Flush 0 Sent Done Done false
186962024-01-24T19:13:39.784Z 15 Acked 1014 Write 7 Sent Done Done false
186972024-01-24T19:13:39.784Z 16 Acked 1015 Flush 0 Sent Done Done false
186982024-01-24T19:13:39.784Z 17 Acked 1016 Write 9 Sent Done Done false
186992024-01-24T19:13:39.784Z 18 Acked 1017 Read 8 Sent Done Done false
187002024-01-24T19:13:39.784Z 19 Acked 1018 Read 7 Sent Done Done false
187012024-01-24T19:13:39.784Z 20 Acked 1019 Read 6 Sent Done Done false
187022024-01-24T19:13:39.784Z 21 Acked 1020 Flush 0 Sent Done Done false
187032024-01-24T19:13:39.784Z 22 Acked 1021 Flush 0 Sent Done Done false
187042024-01-24T19:13:39.785Z 23 Acked 1022 Write 6 Sent Done Done false
187052024-01-24T19:13:39.785Z 24 Acked 1023 Write 5 Sent Done Done false
187062024-01-24T19:13:39.785Z 25 Acked 1024 Read 8 Sent Done Done false
187072024-01-24T19:13:39.785Z 26 Acked 1025 Write 8 Sent Done Done false
187082024-01-24T19:13:39.785Z 27 Acked 1026 Write 3 Sent Done Done false
187092024-01-24T19:13:39.785Z 28 Acked 1027 Read 10 Sent Done Done false
187102024-01-24T19:13:39.785Z 29 Acked 1028 Read 9 Sent Done Done false
187112024-01-24T19:13:39.785Z 30 Acked 1029 Read 8 Sent Done Done false
187122024-01-24T19:13:39.786Z STATES DS:0 DS:1 DS:2 TOTAL
187132024-01-24T19:13:39.786Z New 0 0 0 0
187142024-01-24T19:13:39.786Z Sent 30 0 0 30
187152024-01-24T19:13:39.786Z Done 0 30 30 60
187162024-01-24T19:13:39.786Z Skipped 0 0 0 0
187172024-01-24T19:13:39.786Z Error 0 0 0 0
187182024-01-24T19:13:39.786ZLast Flush: 0 1021 1021
187192024-01-24T19:13:39.786ZDownstairs last five completed:
187202024-01-24T19:13:39.786ZUpstairs last five completed: 30 29 28 27 26
187212024-01-24T19:13:39.786ZWrote out file "/tmp/test_repair_verify.data"
187222024-01-24T19:13:39.791Z
187232024-01-24T19:13:39.794ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187242024-01-24T19:13:39.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187252024-01-24T19:13:39.817Z 0 000-019 168 168 168 295 295 295 F F F
187262024-01-24T19:13:39.817Z 1 020-039 168 174 174 295 305 305 F F F <---
187272024-01-24T19:13:39.817Z 2 040-059 173 173 173 301 301 301 F F F
187282024-01-24T19:13:39.817Z 3 060-079 173 174 174 301 303 303 F F F <---
187292024-01-24T19:13:39.817Z 4 080-099 173 173 173 301 301 301 F F F
187302024-01-24T19:13:39.817Z 5 100-119 167 167 167 292 292 292 F T T <---
187312024-01-24T19:13:39.817Z 6 120-139 172 172 172 299 299 299 F F F
187322024-01-24T19:13:39.817Z 7 140-159 163 163 163 285 285 285 F F F
187332024-01-24T19:13:39.821Z 8 160-179 165 165 165 288 288 288 F F F
187342024-01-24T19:13:39.821Z 9 180-199 165 165 165 288 288 288 F F F
187352024-01-24T19:13:39.821Z 10 200-219 172 172 172 299 299 299 F F F
187362024-01-24T19:13:39.821Z 11 220-239 173 173 173 301 301 301 F F F
187372024-01-24T19:13:39.821Z 12 240-259 173 173 173 301 301 301 F F F
187382024-01-24T19:13:39.821Z 13 260-279 172 172 172 299 299 299 F T T <---
187392024-01-24T19:13:39.821Z 14 280-299 173 173 173 301 301 301 F F F
187402024-01-24T19:13:39.821Z 15 300-319 171 171 171 298 298 298 F F F
187412024-01-24T19:13:39.821Z 16 320-339 168 168 168 293 293 293 F F F
187422024-01-24T19:13:39.821Z 17 340-359 173 173 173 301 301 301 F F F
187432024-01-24T19:13:39.823Z 18 360-379 173 174 174 301 303 303 F F F <---
187442024-01-24T19:13:39.823Z 19 380-399 172 172 172 300 300 300 F F F
187452024-01-24T19:13:39.823Z 20 400-419 172 172 172 300 300 300 F F F
187462024-01-24T19:13:39.823Z 21 420-439 171 171 171 298 298 298 F F F
187472024-01-24T19:13:39.823Z 22 440-459 171 171 171 298 298 298 F F F
187482024-01-24T19:13:39.823Z 23 460-479 173 173 173 301 301 301 F F F
187492024-01-24T19:13:39.823Z 24 480-499 173 174 174 301 302 302 F F F <---
187502024-01-24T19:13:39.823Z 25 500-519 170 170 170 297 297 297 F F F
187512024-01-24T19:13:39.823Z 26 520-539 173 174 174 301 302 302 F T T <---
187522024-01-24T19:13:39.823Z 27 540-559 173 174 174 301 303 303 F F F <---
187532024-01-24T19:13:39.823Z 28 560-579 172 174 174 300 304 304 F F F <---
187542024-01-24T19:13:39.823Z 29 580-599 168 174 174 293 303 303 F T T <---
187552024-01-24T19:13:39.823ZMax gen: 174, Max flush: 305
187562024-01-24T19:13:39.823ZError: Difference in extent metadata found!
187572024-01-24T19:13:39.823ZA Difference in extent metadata is expected here
187582024-01-24T19:13:39.824ZOn loop 87
187592024-01-24T19:13:39.824Z
187602024-01-24T19:13:39.829ZVerifying data now
187612024-01-24T19:13:44.744ZNo change after verify
187622024-01-24T19:13:44.745ZLoop: 87 Downstairs dump after verify (and repair):
187632024-01-24T19:13:44.766ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187642024-01-24T19:13:44.766Z 0 000-019 168 168 168 295 295 295 F F F
187652024-01-24T19:13:44.770Z 1 020-039 174 174 174 305 305 305 F F F
187662024-01-24T19:13:44.770Z 2 040-059 173 173 173 301 301 301 F F F
187672024-01-24T19:13:44.770Z 3 060-079 174 174 174 303 303 303 F F F
187682024-01-24T19:13:44.770Z 4 080-099 173 173 173 301 301 301 F F F
187692024-01-24T19:13:44.770Z 5 100-119 175 175 175 306 306 306 F F F
187702024-01-24T19:13:44.770Z 6 120-139 172 172 172 299 299 299 F F F
187712024-01-24T19:13:44.770Z 7 140-159 163 163 163 285 285 285 F F F
187722024-01-24T19:13:44.770Z 8 160-179 165 165 165 288 288 288 F F F
187732024-01-24T19:13:44.770Z 9 180-199 165 165 165 288 288 288 F F F
187742024-01-24T19:13:44.770Z 10 200-219 172 172 172 299 299 299 F F F
187752024-01-24T19:13:44.770Z 11 220-239 173 173 173 301 301 301 F F F
187762024-01-24T19:13:44.770Z 12 240-259 173 173 173 301 301 301 F F F
187772024-01-24T19:13:44.770Z 13 260-279 175 175 175 306 306 306 F F F
187782024-01-24T19:13:44.770Z 14 280-299 173 173 173 301 301 301 F F F
187792024-01-24T19:13:44.771Z 15 300-319 171 171 171 298 298 298 F F F
187802024-01-24T19:13:44.771Z 16 320-339 168 168 168 293 293 293 F F F
187812024-01-24T19:13:44.771Z 17 340-359 173 173 173 301 301 301 F F F
187822024-01-24T19:13:44.771Z 18 360-379 174 174 174 303 303 303 F F F
187832024-01-24T19:13:44.771Z 19 380-399 172 172 172 300 300 300 F F F
187842024-01-24T19:13:44.771Z 20 400-419 172 172 172 300 300 300 F F F
187852024-01-24T19:13:44.771Z 21 420-439 171 171 171 298 298 298 F F F
187862024-01-24T19:13:44.771Z 22 440-459 171 171 171 298 298 298 F F F
187872024-01-24T19:13:44.771Z 23 460-479 173 173 173 301 301 301 F F F
187882024-01-24T19:13:44.771Z 24 480-499 174 174 174 302 302 302 F F F
187892024-01-24T19:13:44.771Z 25 500-519 170 170 170 297 297 297 F F F
187902024-01-24T19:13:44.771Z 26 520-539 175 175 175 306 306 306 F F F
187912024-01-24T19:13:44.771Z 27 540-559 174 174 174 303 303 303 F F F
187922024-01-24T19:13:44.771Z 28 560-579 174 174 174 304 304 304 F F F
187932024-01-24T19:13:44.771Z 29 580-599 175 175 175 306 306 306 F F F
187942024-01-24T19:13:44.771ZMax gen: 175, Max flush: 306
187952024-01-24T19:13:44.775Z
187962024-01-24T19:13:44.775ZBegin loop 88
187972024-01-24T19:13:44.775ZDownstairs to restart: 1
187982024-01-24T19:13:44.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.480842382Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833}
187992024-01-24T19:13:44.809Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48131167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833}
188002024-01-24T19:13:44.809Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.481414527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833}
188012024-01-24T19:13:44.809Z{"msg":"Crucible ef8ccdfc-a2c2-4bde-9ebb-828401b1496a has session id: 5c29112b-7ef1-421f-8dea-6740b02e2863","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.481453801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188022024-01-24T19:13:44.809Z{"msg":"Upstairs opts: Upstairs UUID: ef8ccdfc-a2c2-4bde-9ebb-828401b1496a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.481512682Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188032024-01-24T19:13:44.809Z{"msg":"Crucible stats registered with UUID: ef8ccdfc-a2c2-4bde-9ebb-828401b1496a","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.481582551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188042024-01-24T19:13:44.809ZCrucible runtime is spawned
188052024-01-24T19:13:44.809Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.481923911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833}
188062024-01-24T19:13:44.809Z{"msg":"ef8ccdfc-a2c2-4bde-9ebb-828401b1496a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482194874Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188072024-01-24T19:13:44.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48233324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188082024-01-24T19:13:44.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482383019Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188092024-01-24T19:13:44.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482426586Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188102024-01-24T19:13:44.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482501028Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188112024-01-24T19:13:44.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482544871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188122024-01-24T19:13:44.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482582106Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188132024-01-24T19:13:44.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482657651Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188142024-01-24T19:13:44.810Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48270373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188152024-01-24T19:13:44.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482747886Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188162024-01-24T19:13:44.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482788621Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188172024-01-24T19:13:44.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482830638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188182024-01-24T19:13:44.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.482924107Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188192024-01-24T19:13:44.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.484088149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188202024-01-24T19:13:44.810Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.484227379Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188212024-01-24T19:13:44.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.484318384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188222024-01-24T19:13:44.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48436343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188232024-01-24T19:13:44.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48500099Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188242024-01-24T19:13:44.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.485281475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188252024-01-24T19:13:44.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48545447Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188262024-01-24T19:13:44.817Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.485566938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188272024-01-24T19:13:44.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.485903642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188282024-01-24T19:13:44.817Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486005776Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188292024-01-24T19:13:44.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486063466Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188302024-01-24T19:13:44.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486282635Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188312024-01-24T19:13:44.817Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486395544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188322024-01-24T19:13:44.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486424523Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188332024-01-24T19:13:44.817Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.486446819Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188342024-01-24T19:13:44.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.4871752Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188352024-01-24T19:13:44.817Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487238581Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188362024-01-24T19:13:44.817Z{"msg":"[0]R flush_numbers[0..12]: [295, 305, 301, 303, 301, 306, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487266451Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188372024-01-24T19:13:44.817Z{"msg":"[0]R generation[0..12]: [168, 174, 173, 174, 173, 175, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487290017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188382024-01-24T19:13:44.817Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487335077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188392024-01-24T19:13:44.818Z{"msg":"[1]R flush_numbers[0..12]: [295, 305, 301, 303, 301, 306, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487362279Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188402024-01-24T19:13:44.818Z{"msg":"[1]R generation[0..12]: [168, 174, 173, 174, 173, 175, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487407714Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188412024-01-24T19:13:44.818Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487435117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188422024-01-24T19:13:44.818Z{"msg":"[2]R flush_numbers[0..12]: [295, 305, 301, 303, 301, 306, 299, 285, 288, 288, 299, 301]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487479743Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188432024-01-24T19:13:44.819Z{"msg":"[2]R generation[0..12]: [168, 174, 173, 174, 173, 175, 172, 163, 165, 165, 172, 173]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487506108Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188442024-01-24T19:13:44.819Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487529659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188452024-01-24T19:13:44.819Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
188462024-01-24T19:13:44.819Z,"time":"2024-01-24T19:13:43.48756151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188472024-01-24T19:13:44.819Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487605736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188482024-01-24T19:13:44.819Z{"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487665663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188492024-01-24T19:13:44.819Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487708724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"":"downstairs","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188502024-01-24T19:13:44.819Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487772484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188512024-01-24T19:13:44.819Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487812615Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"0","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188522024-01-24T19:13:44.819Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487869978Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"1","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188532024-01-24T19:13:44.819Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487912749Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"client":"2","":"downstairs","session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188542024-01-24T19:13:44.819Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.487972294Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188552024-01-24T19:13:44.819Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.48805901Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188562024-01-24T19:13:44.820Z{"msg":"ef8ccdfc-a2c2-4bde-9ebb-828401b1496a is now active with session: 5c29112b-7ef1-421f-8dea-6740b02e2863","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.488102942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188572024-01-24T19:13:44.820Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.488161127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833,"session_id":"5c29112b-7ef1-421f-8dea-6740b02e2863"}
188582024-01-24T19:13:44.820Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:43.488205361Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1833}
188592024-01-24T19:13:44.820ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188602024-01-24T19:13:44.820ZLoading write count information from file "/tmp/test_repair_verify.data"
188612024-01-24T19:13:44.820ZRun Repair workload
188622024-01-24T19:13:44.820Z01/30 Write block 470 len 32768 data: 9 12 16 13 14 15 15 14
188632024-01-24T19:13:44.820Z02/30 Read block 244 len 16384
188642024-01-24T19:13:44.824Z03/30 Read block 266 len 16384
188652024-01-24T19:13:44.824Z04/30 Write block 114 len 28672 data: 7 7 7 6 8 12 11
188662024-01-24T19:13:44.824Z05/30 Read block 321 len 12288
188672024-01-24T19:13:44.824Z06/30 Read block 260 len 24576
188682024-01-24T19:13:44.824Z07/30 Read block 499 len 8192
188692024-01-24T19:13:44.824Z08/30 Read block 312 len 28672
188702024-01-24T19:13:44.827Z09/30 Read block 474 len 36864
188712024-01-24T19:13:44.831Z10/30 Flush
188722024-01-24T19:13:44.834Z11/30 Write block 230 len 28672 data: 11 12 13 15 16 13 15
188732024-01-24T19:13:44.834Z12/30 Read block 447 len 28672
188742024-01-24T19:13:44.838Z13/30 Write block 404 len 12288 data: 8 9 12
188752024-01-24T19:13:44.838Z14/30 Read block 55 len 16384
188762024-01-24T19:13:44.838Z15/30 Write block 272 len 8192 data: 17 18
188772024-01-24T19:13:44.838Z16/30 Write block 257 len 32768 data: 12 11 12 13 12 10 10 13
188782024-01-24T19:13:44.841Z17/30 Write block 427 len 20480 data: 16 16 18 18 17
188792024-01-24T19:13:44.841Z18/30 Flush
188802024-01-24T19:13:44.844Z19/30 Write block 140 len 4096 data: 12
188812024-01-24T19:13:44.844Z20/30 Flush
188822024-01-24T19:13:44.851Z21/30 Write block 95 len 4096 data: 10
188832024-01-24T19:13:44.851Z22/30 Write block 434 len 4096 data: 16
188842024-01-24T19:13:44.851Z23/30 Read block 70 len 20480
188852024-01-24T19:13:44.851Z24/30 Read block 249 len 36864
188862024-01-24T19:13:44.851Z25/30 Read block 461 len 16384
188872024-01-24T19:13:44.855Z26/30 Write block 313 len 24576 data: 13 17 17 17 16 15
188882024-01-24T19:13:44.855Z27/30 Write block 585 len 12288 data: 13 14 14
188892024-01-24T19:13:44.858Z28/30 Write block 224 len 12288 data: 8 10 10
188902024-01-24T19:13:44.858Z29/30 Write block 500 len 8192 data: 12 13
188912024-01-24T19:13:44.858Z30/30 Write block 74 len 36864 data: 8 9 10 12 14 14 14 15 12
188922024-01-24T19:13:44.858Z----------------------------------------------------------------
188932024-01-24T19:13:44.858Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
188942024-01-24T19:13:44.858ZStates: Active Active Active
188952024-01-24T19:13:44.858ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188962024-01-24T19:13:44.858Z 1 Acked 1000 Write 8 Done Done Done false
188972024-01-24T19:13:44.858Z 2 Acked 1001 Read 4 Done Done Done false
188982024-01-24T19:13:44.859Z 3 Acked 1002 Read 4 Done Done Done false
188992024-01-24T19:13:44.859Z 4 Acked 1003 Write 7 Done Done Done false
189002024-01-24T19:13:44.859Z 5 Acked 1004 Read 3 Done Done Done false
189012024-01-24T19:13:44.859Z 6 Acked 1005 Read 6 Done Done Done false
189022024-01-24T19:13:44.859Z 7 Acked 1006 Read 2 Done Done Done false
189032024-01-24T19:13:44.859Z 8 Acked 1007 Read 7 Done Done Done false
189042024-01-24T19:13:44.859Z 9 Acked 1008 Read 9 Done Sent Done false
189052024-01-24T19:13:44.859Z 10 Acked 1009 Flush 0 Done Sent Done false
189062024-01-24T19:13:44.859Z 11 Acked 1010 Write 7 Done Sent Done false
189072024-01-24T19:13:44.863Z 12 Acked 1011 Read 7 Done Sent Done false
189082024-01-24T19:13:44.863Z 13 Acked 1012 Write 3 Done Sent Done false
189092024-01-24T19:13:44.863Z 14 Acked 1013 Read 4 Done Sent Done false
189102024-01-24T19:13:44.863Z 15 Acked 1014 Write 2 Done Sent Done false
189112024-01-24T19:13:44.863Z 16 Acked 1015 Write 8 Done Sent Done false
189122024-01-24T19:13:44.863Z 17 Acked 1016 Write 5 Done Sent Done false
189132024-01-24T19:13:44.863Z 18 Acked 1017 Flush 0 Done Sent Done false
189142024-01-24T19:13:44.863Z 19 Acked 1018 Write 1 Done Sent Done false
189152024-01-24T19:13:44.863Z 20 Acked 1019 Flush 0 Done Sent Done false
189162024-01-24T19:13:44.863Z 21 Acked 1020 Write 1 Done Sent Done false
189172024-01-24T19:13:44.864Z 22 Acked 1021 Write 1 Done Sent Done false
189182024-01-24T19:13:44.864Z 23 Acked 1022 Read 5 Done Sent Done false
189192024-01-24T19:13:44.864Z 24 Acked 1023 Read 9 Done Sent Done false
189202024-01-24T19:13:44.864Z 25 Acked 1024 Read 4 Done Sent Done false
189212024-01-24T19:13:44.864Z 26 Acked 1025 Write 6 Done Sent Done false
189222024-01-24T19:13:44.864Z 27 Acked 1026 Write 3 Done Sent Done false
189232024-01-24T19:13:44.864Z 28 Acked 1027 Write 3 Done Sent Done false
189242024-01-24T19:13:44.864Z 29 Acked 1028 Write 2 Done Sent Done false
189252024-01-24T19:13:44.864Z 30 Acked 1029 Write 9 Sent Sent Sent false
189262024-01-24T19:13:44.864Z STATES DS:0 DS:1 DS:2 TOTAL
189272024-01-24T19:13:44.864Z New 0 0 0 0
189282024-01-24T19:13:44.864Z Sent 1 22 1 24
189292024-01-24T19:13:44.864Z Done 29 8 29 66
189302024-01-24T19:13:44.864Z Skipped 0 0 0 0
189312024-01-24T19:13:44.864Z Error 0 0 0 0
189322024-01-24T19:13:44.864ZLast Flush: 1019 0 1019
189332024-01-24T19:13:44.864ZDownstairs last five completed:
189342024-01-24T19:13:44.864ZUpstairs last five completed: 30 29 28 27 26
189352024-01-24T19:13:44.866ZWrote out file "/tmp/test_repair_verify.data"
189362024-01-24T19:13:44.871Z
189372024-01-24T19:13:45.282ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189382024-01-24T19:13:45.333ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189392024-01-24T19:13:45.333Z 0 000-019 168 168 168 295 295 295 F F F
189402024-01-24T19:13:45.333Z 1 020-039 174 174 174 305 305 305 F F F
189412024-01-24T19:13:45.333Z 2 040-059 173 173 173 301 301 301 F F F
189422024-01-24T19:13:45.333Z 3 060-079 174 174 174 303 303 303 T F T <---
189432024-01-24T19:13:45.333Z 4 080-099 173 173 173 301 301 301 T F T <---
189442024-01-24T19:13:45.333Z 5 100-119 176 175 176 307 306 307 F T F <---
189452024-01-24T19:13:45.333Z 6 120-139 176 172 176 307 299 307 F T F <---
189462024-01-24T19:13:45.333Z 7 140-159 176 163 176 309 285 309 F F F <---
189472024-01-24T19:13:45.334Z 8 160-179 165 165 165 288 288 288 F F F
189482024-01-24T19:13:45.334Z 9 180-199 165 165 165 288 288 288 F F F
189492024-01-24T19:13:45.334Z 10 200-219 172 172 172 299 299 299 F F F
189502024-01-24T19:13:45.334Z 11 220-239 176 173 176 308 301 308 T F T <---
189512024-01-24T19:13:45.335Z 12 240-259 176 173 176 308 301 308 F F F <---
189522024-01-24T19:13:45.335Z 13 260-279 176 175 176 308 306 308 F F F <---
189532024-01-24T19:13:45.335Z 14 280-299 173 173 173 301 301 301 F F F
189542024-01-24T19:13:45.335Z 15 300-319 171 171 171 298 298 298 T F T <---
189552024-01-24T19:13:45.335Z 16 320-339 168 168 168 293 293 293 F F F
189562024-01-24T19:13:45.335Z 17 340-359 173 173 173 301 301 301 F F F
189572024-01-24T19:13:45.336Z 18 360-379 174 174 174 303 303 303 F F F
189582024-01-24T19:13:45.336Z 19 380-399 172 172 172 300 300 300 F F F
189592024-01-24T19:13:45.336Z 20 400-419 176 172 176 308 300 308 F F F <---
189602024-01-24T19:13:45.336Z 21 420-439 176 171 176 308 298 308 T F T <---
189612024-01-24T19:13:45.336Z 22 440-459 171 171 171 298 298 298 F F F
189622024-01-24T19:13:45.336Z 23 460-479 176 173 176 307 301 307 F T F <---
189632024-01-24T19:13:45.336Z 24 480-499 174 174 174 302 302 302 F F F
189642024-01-24T19:13:45.337Z 25 500-519 170 170 170 297 297 297 T F T <---
189652024-01-24T19:13:45.337Z 26 520-539 175 175 175 306 306 306 F F F
189662024-01-24T19:13:45.337Z 27 540-559 174 174 174 303 303 303 F F F
189672024-01-24T19:13:45.337Z 28 560-579 174 174 174 304 304 304 F F F
189682024-01-24T19:13:45.337Z 29 580-599 175 175 175 306 306 306 T F T <---
189692024-01-24T19:13:45.337ZMax gen: 176, Max flush: 309
189702024-01-24T19:13:45.337ZError: Difference in extent metadata found!
189712024-01-24T19:13:45.337ZA Difference in extent metadata is expected here
189722024-01-24T19:13:45.337ZOn loop 88
189732024-01-24T19:13:45.337Z
189742024-01-24T19:13:45.337ZVerifying data now
189752024-01-24T19:13:51.143ZNo change after verify
189762024-01-24T19:13:51.143ZLoop: 88 Downstairs dump after verify (and repair):
189772024-01-24T19:13:51.169ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189782024-01-24T19:13:51.169Z 0 000-019 168 168 168 295 295 295 F F F
189792024-01-24T19:13:51.175Z 1 020-039 174 174 174 305 305 305 F F F
189802024-01-24T19:13:51.176Z 2 040-059 173 173 173 301 301 301 F F F
189812024-01-24T19:13:51.176Z 3 060-079 177 177 177 310 310 310 F F F
189822024-01-24T19:13:51.176Z 4 080-099 177 177 177 310 310 310 F F F
189832024-01-24T19:13:51.176Z 5 100-119 176 176 176 307 307 307 F F F
189842024-01-24T19:13:51.177Z 6 120-139 176 176 176 307 307 307 F F F
189852024-01-24T19:13:51.177Z 7 140-159 176 176 176 309 309 309 F F F
189862024-01-24T19:13:51.177Z 8 160-179 165 165 165 288 288 288 F F F
189872024-01-24T19:13:51.177Z 9 180-199 165 165 165 288 288 288 F F F
189882024-01-24T19:13:51.178Z 10 200-219 172 172 172 299 299 299 F F F
189892024-01-24T19:13:51.178Z 11 220-239 177 177 177 310 310 310 F F F
189902024-01-24T19:13:51.178Z 12 240-259 176 176 176 308 308 308 F F F
189912024-01-24T19:13:51.178Z 13 260-279 176 176 176 308 308 308 F F F
189922024-01-24T19:13:51.178Z 14 280-299 173 173 173 301 301 301 F F F
189932024-01-24T19:13:51.179Z 15 300-319 177 177 177 310 310 310 F F F
189942024-01-24T19:13:51.179Z 16 320-339 168 168 168 293 293 293 F F F
189952024-01-24T19:13:51.179Z 17 340-359 173 173 173 301 301 301 F F F
189962024-01-24T19:13:51.181Z 18 360-379 174 174 174 303 303 303 F F F
189972024-01-24T19:13:51.181Z 19 380-399 172 172 172 300 300 300 F F F
189982024-01-24T19:13:51.181Z 20 400-419 176 176 176 308 308 308 F F F
189992024-01-24T19:13:51.181Z 21 420-439 177 177 177 310 310 310 F F F
190002024-01-24T19:13:51.181Z 22 440-459 171 171 171 298 298 298 F F F
190012024-01-24T19:13:51.181Z 23 460-479 176 176 176 307 307 307 F F F
190022024-01-24T19:13:51.181Z 24 480-499 174 174 174 302 302 302 F F F
190032024-01-24T19:13:51.181Z 25 500-519 177 177 177 310 310 310 F F F
190042024-01-24T19:13:51.181Z 26 520-539 175 175 175 306 306 306 F F F
190052024-01-24T19:13:51.181Z 27 540-559 174 174 174 303 303 303 F F F
190062024-01-24T19:13:51.182Z 28 560-579 174 174 174 304 304 304 F F F
190072024-01-24T19:13:51.182Z 29 580-599 177 177 177 310 310 310 F F F
190082024-01-24T19:13:51.182ZMax gen: 177, Max flush: 310
190092024-01-24T19:13:51.182Z
190102024-01-24T19:13:51.182ZBegin loop 89
190112024-01-24T19:13:51.182ZDownstairs to restart: 1
190122024-01-24T19:13:51.199Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.877107272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841}
190132024-01-24T19:13:51.206Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.877622969Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841}
190142024-01-24T19:13:51.206ZCrucible runtime is spawned
190152024-01-24T19:13:51.206Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.877867435Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841}
190162024-01-24T19:13:51.207Z{"msg":"Crucible 83d5eb69-9aa2-48a6-a4c3-e03ca00f9198 has session id: d218689e-f444-4811-befa-13ba21b80ad9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.8779428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190172024-01-24T19:13:51.207Z{"msg":"Upstairs opts: Upstairs UUID: 83d5eb69-9aa2-48a6-a4c3-e03ca00f9198, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878004544Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190182024-01-24T19:13:51.207Z{"msg":"Crucible stats registered with UUID: 83d5eb69-9aa2-48a6-a4c3-e03ca00f9198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878050307Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190192024-01-24T19:13:51.207Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878122847Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841}
190202024-01-24T19:13:51.207Z{"msg":"83d5eb69-9aa2-48a6-a4c3-e03ca00f9198 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878311284Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190212024-01-24T19:13:51.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878380345Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190222024-01-24T19:13:51.207Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.87849676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190232024-01-24T19:13:51.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878541603Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190242024-01-24T19:13:51.207Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878781739Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190252024-01-24T19:13:51.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878835016Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190262024-01-24T19:13:51.208Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878910942Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190272024-01-24T19:13:51.208Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.878952081Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190282024-01-24T19:13:51.209Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.87902831Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190292024-01-24T19:13:51.209Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.879075472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190302024-01-24T19:13:51.209Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.879157104Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190312024-01-24T19:13:51.209Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.879260529Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190322024-01-24T19:13:51.209Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.879302871Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190332024-01-24T19:13:51.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.880088537Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190342024-01-24T19:13:51.210Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.880228138Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190352024-01-24T19:13:51.210Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.880285694Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190362024-01-24T19:13:51.211Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.880363337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190372024-01-24T19:13:51.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.88044824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190382024-01-24T19:13:51.211Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.880533885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190392024-01-24T19:13:51.217Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.88134993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190402024-01-24T19:13:51.217Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.881529506Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190412024-01-24T19:13:51.217Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.881875034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190422024-01-24T19:13:51.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882326309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190432024-01-24T19:13:51.218Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882452208Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190442024-01-24T19:13:51.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.88249102Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190452024-01-24T19:13:51.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882538951Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190462024-01-24T19:13:51.218Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882566069Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190472024-01-24T19:13:51.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882589147Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190482024-01-24T19:13:51.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882851372Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190492024-01-24T19:13:51.218Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882888587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190502024-01-24T19:13:51.218Z{"msg":"[0]R flush_numbers[0..12]: [295, 305, 301, 310, 310, 307, 307, 309, 288, 288, 299, 310]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.882913078Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190512024-01-24T19:13:51.218Z{"msg":"[0]R generation[0..12]: [168, 174, 173, 177, 177, 176, 176, 176, 165, 165, 172, 177]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883028487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190522024-01-24T19:13:51.218Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883055568Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190532024-01-24T19:13:51.218Z{"msg":"[1]R flush_numbers[0..12]: [295, 305, 301, 310, 310, 307, 307, 309, 288, 288, 299, 310]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883078623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190542024-01-24T19:13:51.218Z{"msg":"[1]R generation[0..12]: [168, 174, 173, 177, 177, 176, 176, 176, 165, 165, 172, 177]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883121891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190552024-01-24T19:13:51.218Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883148963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190562024-01-24T19:13:51.218Z{"msg":"[2]R flush_numbers[0..12]: [295, 305, 301, 310, 310, 307, 307, 309, 288, 288, 299, 310]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883172319Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190572024-01-24T19:13:51.218Z{"msg":"[2]R generation[0..12]: [168, 174, 173, 177, 177, 176, 176, 176, 165, 165, 172, 177]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.88322292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190582024-01-24T19:13:51.218Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883249686Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190592024-01-24T19:13:51.218Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883271273Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190602024-01-24T19:13:51.218Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883293032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190612024-01-24T19:13:51.218Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
190622024-01-24T19:13:51.218Z30,"time":"2024-01-24T19:13:49.883342191Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190632024-01-24T19:13:51.218Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883367245Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"":"downstairs","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190642024-01-24T19:13:51.219Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883410661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190652024-01-24T19:13:51.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883433885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"0","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190662024-01-24T19:13:51.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883477293Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"1","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190672024-01-24T19:13:51.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883502214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"client":"2","":"downstairs","session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190682024-01-24T19:13:51.219Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.88355274Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190692024-01-24T19:13:51.219Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883576333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190702024-01-24T19:13:51.219Z{"msg":"83d5eb69-9aa2-48a6-a4c3-e03ca00f9198 is now active with session: d218689e-f444-4811-befa-13ba21b80ad9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883597974Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190712024-01-24T19:13:51.219Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883618724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841,"session_id":"d218689e-f444-4811-befa-13ba21b80ad9"}
190722024-01-24T19:13:51.219Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:49.883659188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1841}
190732024-01-24T19:13:51.219ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
190742024-01-24T19:13:51.219ZLoading write count information from file "/tmp/test_repair_verify.data"
190752024-01-24T19:13:51.219ZRun Repair workload
190762024-01-24T19:13:51.219Z01/30 Write block 405 len 20480 data: 10 13 12 14 8
190772024-01-24T19:13:51.219Z02/30 Write block 463 len 40960 data: 10 9 10 10 11 10 8 10 13 17
190782024-01-24T19:13:51.219Z03/30 Write block 173 len 8192 data: 15 18
190792024-01-24T19:13:51.219Z04/30 Read block 511 len 36864
190802024-01-24T19:13:51.219Z05/30 Flush
190812024-01-24T19:13:51.223Z06/30 Write block 467 len 8192 data: 12 11
190822024-01-24T19:13:51.223Z07/30 Flush
190832024-01-24T19:13:51.223Z08/30 Write block 330 len 12288 data: 14 12 14
190842024-01-24T19:13:51.223Z09/30 Write block 71 len 8192 data: 8 9
190852024-01-24T19:13:51.223Z10/30 Read block 443 len 32768
190862024-01-24T19:13:51.226Z11/30 Read block 337 len 16384
190872024-01-24T19:13:51.226Z12/30 Write block 373 len 40960 data: 16 15 18 16 16 11 11 14 14 12
190882024-01-24T19:13:51.229Z13/30 Flush
190892024-01-24T19:13:51.232Z14/30 Write block 221 len 28672 data: 9 11 7 9 11 11 11
190902024-01-24T19:13:51.232Z15/30 Write block 239 len 24576 data: 13 12 11 13 18 18
190912024-01-24T19:13:51.235Z16/30 Write block 251 len 40960 data: 12 8 8 10 13 14 13 12 13 14
190922024-01-24T19:13:51.235Z17/30 Write block 388 len 40960 data: 10 8 8 9 8 10 11 12 13 13
190932024-01-24T19:13:51.235Z18/30 Write block 350 len 4096 data: 11
190942024-01-24T19:13:51.238Z19/30 Read block 45 len 12288
190952024-01-24T19:13:51.238Z20/30 Read block 0 len 32768
190962024-01-24T19:13:51.242Z21/30 Read block 570 len 32768
190972024-01-24T19:13:51.242Z22/30 Read block 56 len 16384
190982024-01-24T19:13:51.245Z23/30 Read block 276 len 8192
190992024-01-24T19:13:51.245Z24/30 Write block 175 len 12288 data: 14 12 10
191002024-01-24T19:13:51.248Z25/30 Write block 381 len 40960 data: 15 13 13 10 8 8 9 11 9 9
191012024-01-24T19:13:51.248Z26/30 Write block 158 len 16384 data: 17 12 12 20
191022024-01-24T19:13:51.248Z27/30 Write block 184 len 12288 data: 11 12 12
191032024-01-24T19:13:51.252Z28/30 Read block 409 len 20480
191042024-01-24T19:13:51.252Z29/30 Write block 282 len 20480 data: 19 17 14 15 14
191052024-01-24T19:13:51.252Z30/30 Write block 307 len 4096 data: 9
191062024-01-24T19:13:51.256Z----------------------------------------------------------------
191072024-01-24T19:13:51.256Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
191082024-01-24T19:13:51.256ZStates: Active Active Active
191092024-01-24T19:13:51.256ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191102024-01-24T19:13:51.256Z 1 Acked 1000 Write 5 Done Done Done false
191112024-01-24T19:13:51.256Z 2 Acked 1001 Write 10 Done Done Done false
191122024-01-24T19:13:51.256Z 3 Acked 1002 Write 2 Done Sent Done false
191132024-01-24T19:13:51.256Z 4 Acked 1003 Read 9 Done Sent Done false
191142024-01-24T19:13:51.256Z 5 Acked 1004 Flush 0 Done Sent Done false
191152024-01-24T19:13:51.256Z 6 Acked 1005 Write 2 Done Sent Done false
191162024-01-24T19:13:51.256Z 7 Acked 1006 Flush 0 Done Sent Done false
191172024-01-24T19:13:51.256Z 8 Acked 1007 Write 3 Done Sent Done false
191182024-01-24T19:13:51.256Z 9 Acked 1008 Write 2 Done Sent Done false
191192024-01-24T19:13:51.256Z 10 Acked 1009 Read 8 Done Sent Done false
191202024-01-24T19:13:51.257Z 11 Acked 1010 Read 4 Done Sent Done false
191212024-01-24T19:13:51.257Z 12 Acked 1011 Write 10 Done Sent Done false
191222024-01-24T19:13:51.257Z 13 Acked 1012 Flush 0 Done Sent Done false
191232024-01-24T19:13:51.257Z 14 Acked 1013 Write 7 Done Sent Done false
191242024-01-24T19:13:51.257Z 15 Acked 1014 Write 6 Done Sent Done false
191252024-01-24T19:13:51.257Z 16 Acked 1015 Write 10 Done Sent Done false
191262024-01-24T19:13:51.257Z 17 Acked 1016 Write 10 Done Sent Done false
191272024-01-24T19:13:51.260Z 18 Acked 1017 Write 1 Done Sent Done false
191282024-01-24T19:13:51.260Z 19 Acked 1018 Read 3 Done Sent Done false
191292024-01-24T19:13:51.260Z 20 Acked 1019 Read 8 Done Sent Done false
191302024-01-24T19:13:51.260Z 21 Acked 1020 Read 8 Done Sent Done false
191312024-01-24T19:13:51.261Z 22 Acked 1021 Read 4 Done Sent Done false
191322024-01-24T19:13:51.261Z 23 Acked 1022 Read 2 Done Sent Done false
191332024-01-24T19:13:51.261Z 24 Acked 1023 Write 3 Done Sent Done false
191342024-01-24T19:13:51.261Z 25 Acked 1024 Write 10 Done Sent Done false
191352024-01-24T19:13:51.261Z 26 Acked 1025 Write 4 Done Sent Done false
191362024-01-24T19:13:51.261Z 27 Acked 1026 Write 3 Done Sent Done false
191372024-01-24T19:13:51.261Z 28 Acked 1027 Read 5 Done Sent Done false
191382024-01-24T19:13:51.261Z 29 Acked 1028 Write 5 Done Sent Done false
191392024-01-24T19:13:51.261Z 30 Acked 1029 Write 1 Sent Sent Sent false
191402024-01-24T19:13:51.261Z STATES DS:0 DS:1 DS:2 TOTAL
191412024-01-24T19:13:51.261Z New 0 0 0 0
191422024-01-24T19:13:51.261Z Sent 1 28 1 30
191432024-01-24T19:13:51.261Z Done 29 2 29 60
191442024-01-24T19:13:51.261Z Skipped 0 0 0 0
191452024-01-24T19:13:51.261Z Error 0 0 0 0
191462024-01-24T19:13:51.261ZLast Flush: 1012 0 1012
191472024-01-24T19:13:51.261ZDownstairs last five completed:
191482024-01-24T19:13:51.261ZUpstairs last five completed: 30 29 28 27 26
191492024-01-24T19:13:51.261ZWrote out file "/tmp/test_repair_verify.data"
191502024-01-24T19:13:51.265Z
191512024-01-24T19:13:51.265ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191522024-01-24T19:13:51.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191532024-01-24T19:13:51.292Z 0 000-019 168 168 168 295 295 295 F F F
191542024-01-24T19:13:51.292Z 1 020-039 174 174 174 305 305 305 F F F
191552024-01-24T19:13:51.292Z 2 040-059 173 173 173 301 301 301 F F F
191562024-01-24T19:13:51.292Z 3 060-079 178 177 178 313 310 313 F F F <---
191572024-01-24T19:13:51.297Z 4 080-099 177 177 177 310 310 310 F F F
191582024-01-24T19:13:51.297Z 5 100-119 176 176 176 307 307 307 F F F
191592024-01-24T19:13:51.297Z 6 120-139 176 176 176 307 307 307 F F F
191602024-01-24T19:13:51.297Z 7 140-159 176 176 176 309 309 309 T F T <---
191612024-01-24T19:13:51.297Z 8 160-179 178 165 178 311 288 311 T F T <---
191622024-01-24T19:13:51.298Z 9 180-199 165 165 165 288 288 288 T F T <---
191632024-01-24T19:13:51.298Z 10 200-219 172 172 172 299 299 299 F F F
191642024-01-24T19:13:51.298Z 11 220-239 177 177 177 310 310 310 T F T <---
191652024-01-24T19:13:51.298Z 12 240-259 176 176 176 308 308 308 T F T <---
191662024-01-24T19:13:51.298Z 13 260-279 176 176 176 308 308 308 T F T <---
191672024-01-24T19:13:51.299Z 14 280-299 173 173 173 301 301 301 T F T <---
191682024-01-24T19:13:51.299Z 15 300-319 177 177 177 310 310 310 T F T <---
191692024-01-24T19:13:51.299Z 16 320-339 178 168 178 313 293 313 F F F <---
191702024-01-24T19:13:51.299Z 17 340-359 173 173 173 301 301 301 T F T <---
191712024-01-24T19:13:51.299Z 18 360-379 178 174 178 313 303 313 F F F <---
191722024-01-24T19:13:51.299Z 19 380-399 178 172 178 313 300 313 T F T <---
191732024-01-24T19:13:51.300Z 20 400-419 178 176 178 311 308 311 F T F <---
191742024-01-24T19:13:51.300Z 21 420-439 177 177 177 310 310 310 F F F
191752024-01-24T19:13:51.301Z 22 440-459 171 171 171 298 298 298 F F F
191762024-01-24T19:13:51.301Z 23 460-479 178 176 178 312 307 312 F T F <---
191772024-01-24T19:13:51.302Z 24 480-499 174 174 174 302 302 302 F F F
191782024-01-24T19:13:51.302Z 25 500-519 177 177 177 310 310 310 F F F
191792024-01-24T19:13:51.302Z 26 520-539 175 175 175 306 306 306 F F F
191802024-01-24T19:13:51.302Z 27 540-559 174 174 174 303 303 303 F F F
191812024-01-24T19:13:51.302Z 28 560-579 174 174 174 304 304 304 F F F
191822024-01-24T19:13:51.303Z 29 580-599 177 177 177 310 310 310 F F F
191832024-01-24T19:13:51.303ZMax gen: 178, Max flush: 313
191842024-01-24T19:13:51.303ZError: Difference in extent metadata found!
191852024-01-24T19:13:51.317ZA Difference in extent metadata is expected here
191862024-01-24T19:13:51.317ZOn loop 89
191872024-01-24T19:13:51.317Z
191882024-01-24T19:13:51.317ZVerifying data now
191892024-01-24T19:13:57.617ZNo change after verify
191902024-01-24T19:13:57.617ZLoop: 89 Downstairs dump after verify (and repair):
191912024-01-24T19:13:57.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191922024-01-24T19:13:57.639Z 0 000-019 168 168 168 295 295 295 F F F
191932024-01-24T19:13:57.640Z 1 020-039 174 174 174 305 305 305 F F F
191942024-01-24T19:13:57.640Z 2 040-059 173 173 173 301 301 301 F F F
191952024-01-24T19:13:57.644Z 3 060-079 178 178 178 313 313 313 F F F
191962024-01-24T19:13:57.644Z 4 080-099 177 177 177 310 310 310 F F F
191972024-01-24T19:13:57.644Z 5 100-119 176 176 176 307 307 307 F F F
191982024-01-24T19:13:57.644Z 6 120-139 176 176 176 307 307 307 F F F
191992024-01-24T19:13:57.644Z 7 140-159 179 179 179 314 314 314 F F F
192002024-01-24T19:13:57.644Z 8 160-179 179 179 179 314 314 314 F F F
192012024-01-24T19:13:57.644Z 9 180-199 179 179 179 314 314 314 F F F
192022024-01-24T19:13:57.644Z 10 200-219 172 172 172 299 299 299 F F F
192032024-01-24T19:13:57.644Z 11 220-239 179 179 179 314 314 314 F F F
192042024-01-24T19:13:57.644Z 12 240-259 179 179 179 314 314 314 F F F
192052024-01-24T19:13:57.644Z 13 260-279 179 179 179 314 314 314 F F F
192062024-01-24T19:13:57.644Z 14 280-299 179 179 179 314 314 314 F F F
192072024-01-24T19:13:57.644Z 15 300-319 179 179 179 314 314 314 F F F
192082024-01-24T19:13:57.644Z 16 320-339 178 178 178 313 313 313 F F F
192092024-01-24T19:13:57.644Z 17 340-359 179 179 179 314 314 314 F F F
192102024-01-24T19:13:57.644Z 18 360-379 178 178 178 313 313 313 F F F
192112024-01-24T19:13:57.644Z 19 380-399 179 179 179 314 314 314 F F F
192122024-01-24T19:13:57.644Z 20 400-419 178 178 178 311 311 311 F F F
192132024-01-24T19:13:57.644Z 21 420-439 177 177 177 310 310 310 F F F
192142024-01-24T19:13:57.645Z 22 440-459 171 171 171 298 298 298 F F F
192152024-01-24T19:13:57.645Z 23 460-479 178 178 178 312 312 312 F F F
192162024-01-24T19:13:57.645Z 24 480-499 174 174 174 302 302 302 F F F
192172024-01-24T19:13:57.645Z 25 500-519 177 177 177 310 310 310 F F F
192182024-01-24T19:13:57.645Z 26 520-539 175 175 175 306 306 306 F F F
192192024-01-24T19:13:57.645Z 27 540-559 174 174 174 303 303 303 F F F
192202024-01-24T19:13:57.646Z 28 560-579 174 174 174 304 304 304 F F F
192212024-01-24T19:13:57.646Z 29 580-599 177 177 177 310 310 310 F F F
192222024-01-24T19:13:57.646ZMax gen: 179, Max flush: 314
192232024-01-24T19:13:57.650Z
192242024-01-24T19:13:57.650ZBegin loop 90
192252024-01-24T19:13:57.650ZDownstairs to restart: 1
192262024-01-24T19:13:57.678Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356075318Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849}
192272024-01-24T19:13:57.683Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356558539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849}
192282024-01-24T19:13:57.683Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356642457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849}
192292024-01-24T19:13:57.683Z{"msg":"Crucible ef44cd62-0f6c-4574-bca1-027bf48ac68c has session id: fa864437-16f3-4d13-a461-cf60c0f34d13","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356721324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192302024-01-24T19:13:57.683Z{"msg":"Upstairs opts: Upstairs UUID: ef44cd62-0f6c-4574-bca1-027bf48ac68c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356826384Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"Crucible runtime is spawned
192312024-01-24T19:13:57.683Z}
192322024-01-24T19:13:57.683Z{"msg":"Crucible stats registered with UUID: ef44cd62-0f6c-4574-bca1-027bf48ac68c","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.356953696Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192332024-01-24T19:13:57.683Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357027879Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849}
192342024-01-24T19:13:57.683Z{"msg":"ef44cd62-0f6c-4574-bca1-027bf48ac68c active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.35734487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192352024-01-24T19:13:57.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357457149Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192362024-01-24T19:13:57.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357485298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192372024-01-24T19:13:57.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357524908Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192382024-01-24T19:13:57.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357584394Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192392024-01-24T19:13:57.683Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357614135Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192402024-01-24T19:13:57.683Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357667014Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192412024-01-24T19:13:57.683Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357693426Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192422024-01-24T19:13:57.683Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357766623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192432024-01-24T19:13:57.683Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.357836503Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192442024-01-24T19:13:57.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.358049129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192452024-01-24T19:13:57.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.358158634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192462024-01-24T19:13:57.683Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.358206236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192472024-01-24T19:13:57.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.359412009Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192482024-01-24T19:13:57.683Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.359503836Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192492024-01-24T19:13:57.684Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.359548232Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192502024-01-24T19:13:57.684Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.359588158Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192512024-01-24T19:13:57.690Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.360554797Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192522024-01-24T19:13:57.690Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.360624841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192532024-01-24T19:13:57.691Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.36082127Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192542024-01-24T19:13:57.691Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.361045163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192552024-01-24T19:13:57.691Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.361642292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192562024-01-24T19:13:57.691Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.361935873Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192572024-01-24T19:13:57.691Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362017487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192582024-01-24T19:13:57.691Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362062229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192592024-01-24T19:13:57.691Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362130774Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192602024-01-24T19:13:57.691Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362195219Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192612024-01-24T19:13:57.691Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362223197Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192622024-01-24T19:13:57.692Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362489463Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192632024-01-24T19:13:57.695Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.36259168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192642024-01-24T19:13:57.695Z{"msg":"[0]R flush_numbers[0..12]: [295, 305, 301, 313, 310, 307, 307, 314, 314, 314, 299, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362666911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192652024-01-24T19:13:57.695Z{"msg":"[0]R generation[0..12]: [168, 174, 173, 178, 177, 176, 176, 179, 179, 179, 172, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362710074Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192662024-01-24T19:13:57.696Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.36277575Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192672024-01-24T19:13:57.696Z{"msg":"[1]R flush_numbers[0..12]: [295, 305, 301, 313, 310, 307, 307, 314, 314, 314, 299, 314]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
192682024-01-24T19:13:57.696Z,"time":"2024-01-24T19:13:56.362822233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192692024-01-24T19:13:57.696Z{"msg":"[1]R generation[0..12]: [168, 174, 173, 178, 177, 176, 176, 179, 179, 179, 172, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362925179Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192702024-01-24T19:13:57.696Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.362994405Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192712024-01-24T19:13:57.696Z{"msg":"[2]R flush_numbers[0..12]: [295, 305, 301, 313, 310, 307, 307, 314, 314, 314, 299, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363040374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192722024-01-24T19:13:57.696Z{"msg":"[2]R generation[0..12]: [168, 174, 173, 178, 177, 176, 176, 179, 179, 179, 172, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363104366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192732024-01-24T19:13:57.697Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363150402Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192742024-01-24T19:13:57.697Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363211805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192752024-01-24T19:13:57.697Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363254449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192762024-01-24T19:13:57.697Z{"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363323013Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192772024-01-24T19:13:57.697Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363385494Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"":"downstairs","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192782024-01-24T19:13:57.697Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363429301Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192792024-01-24T19:13:57.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363486003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"0","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192802024-01-24T19:13:57.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363528762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"1","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192812024-01-24T19:13:57.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363588792Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"client":"2","":"downstairs","session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192822024-01-24T19:13:57.698Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.36363114Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
192832024-01-24T19:13:57.699Zip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192842024-01-24T19:13:57.699Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363733349Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192852024-01-24T19:13:57.699Z{"msg":"ef44cd62-0f6c-4574-bca1-027bf48ac68c is now active with session: fa864437-16f3-4d13-a461-cf60c0f34d13","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363792765Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192862024-01-24T19:13:57.699Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363833663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849,"session_id":"fa864437-16f3-4d13-a461-cf60c0f34d13"}
192872024-01-24T19:13:57.699Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:13:56.363888077Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1849}
192882024-01-24T19:13:57.699ZLoading write count information from file "/tmp/test_repair_verify.data"
192892024-01-24T19:13:57.699ZRun Repair workload
192902024-01-24T19:13:57.700Z01/30 Write block 521 len 4096 data: 13
192912024-01-24T19:13:57.706Z02/30 Write block 58 len 24576 data: 11 8 10 10 9 11
192922024-01-24T19:13:57.706Z03/30 Write block 344 len 16384 data: 14 14 12 14
192932024-01-24T19:13:57.706Z04/30 Write block 215 len 4096 data: 8
192942024-01-24T19:13:57.706Z05/30 Flush
192952024-01-24T19:13:57.706Z06/30 Write block 383 len 32768 data: 14 11 9 9 10 12 10 10
192962024-01-24T19:13:57.706Z07/30 Read block 39 len 16384
192972024-01-24T19:13:57.706Z08/30 Write block 447 len 40960 data: 7 8 8 7 9 10 12 11 10 13
192982024-01-24T19:13:57.706Z09/30 Write block 524 len 40960 data: 15 16 16 14 19 17 16 12 9 13
192992024-01-24T19:13:57.706Z10/30 Read block 196 len 8192
193002024-01-24T19:13:57.706Z11/30 Read block 29 len 40960
193012024-01-24T19:13:57.710Z12/30 Write block 304 len 28672 data: 6 8 8 10 8 9 11
193022024-01-24T19:13:57.711Z13/30 Write block 392 len 12288 data: 9 11 12
193032024-01-24T19:13:57.711Z14/30 Write block 113 len 40960 data: 10 8 8 8 7 9 13 12 13 14
193042024-01-24T19:13:57.711Z15/30 Read block 277 len 4096
193052024-01-24T19:13:57.716Z16/30 Read block 412 len 20480
193062024-01-24T19:13:57.716Z17/30 Read block 60 len 4096
193072024-01-24T19:13:57.716Z18/30 Write block 304 len 8192 data: 7 9
193082024-01-24T19:13:57.716Z19/30 Write block 583 len 24576 data: 14 15 14 15 15 14
193092024-01-24T19:13:57.716Z20/30 Read block 122 len 40960
193102024-01-24T19:13:57.719Z21/30 Write block 555 len 20480 data: 11 12 14 14 15
193112024-01-24T19:13:57.720Z22/30 Read block 76 len 36864
193122024-01-24T19:13:57.724Z23/30 Flush
193132024-01-24T19:13:57.730Z24/30 Write block 359 len 40960 data: 12 12 17 16 17 18 18 18 19 17
193142024-01-24T19:13:57.731Z25/30 Flush
193152024-01-24T19:13:57.731Z26/30 Write block 107 len 40960 data: 19 19 20 19 17 13 11 9 9 9
193162024-01-24T19:13:57.735Z27/30 Write block 162 len 16384 data: 19 20 16 21
193172024-01-24T19:13:57.735Z28/30 Write block 524 len 32768 data: 16 17 17 15 20 18 17 13
193182024-01-24T19:13:57.735Z29/30 Write block 150 len 20480 data: 15 17 19 16 15
193192024-01-24T19:13:57.738Z30/30 Read block 53 len 24576
193202024-01-24T19:13:57.743Z----------------------------------------------------------------
193212024-01-24T19:13:57.743Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
193222024-01-24T19:13:57.743ZStates: Active Active Active
193232024-01-24T19:13:57.743ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193242024-01-24T19:13:57.743Z 1 Acked 1000 Write 1 Done Done Done false
193252024-01-24T19:13:57.743Z 2 Acked 1001 Write 6 Done Done Done false
193262024-01-24T19:13:57.743Z 3 Acked 1002 Write 4 Done Sent Done false
193272024-01-24T19:13:57.744Z 4 Acked 1003 Write 1 Done Sent Done false
193282024-01-24T19:13:57.744Z 5 Acked 1004 Flush 0 Done Sent Done false
193292024-01-24T19:13:57.744Z 6 Acked 1005 Write 8 Done Sent Done false
193302024-01-24T19:13:57.744Z 7 Acked 1006 Read 4 Done Sent Done false
193312024-01-24T19:13:57.744Z 8 Acked 1007 Write 10 Done Sent Done false
193322024-01-24T19:13:57.744Z 9 Acked 1008 Write 10 Done Sent Done false
193332024-01-24T19:13:57.744Z 10 Acked 1009 Read 2 Done Sent Done false
193342024-01-24T19:13:57.744Z 11 Acked 1010 Read 10 Done Sent Done false
193352024-01-24T19:13:57.745Z 12 Acked 1011 Write 7 Done Sent Done false
193362024-01-24T19:13:57.745Z 13 Acked 1012 Write 3 Done Sent Done false
193372024-01-24T19:13:57.745Z 14 Acked 1013 Write 10 Done Sent Done false
193382024-01-24T19:13:57.745Z 15 Acked 1014 Read 1 Done Sent Done false
193392024-01-24T19:13:57.745Z 16 Acked 1015 Read 5 Done Sent Done false
193402024-01-24T19:13:57.745Z 17 Acked 1016 Read 1 Done Sent Done false
193412024-01-24T19:13:57.745Z 18 Acked 1017 Write 2 Done Sent Done false
193422024-01-24T19:13:57.745Z 19 Acked 1018 Write 6 Done Sent Done false
193432024-01-24T19:13:57.745Z 20 Acked 1019 Read 10 Done Sent Done false
193442024-01-24T19:13:57.745Z 21 Acked 1020 Write 5 Done Sent Done false
193452024-01-24T19:13:57.745Z 22 Acked 1021 Read 9 Done Sent Done false
193462024-01-24T19:13:57.745Z 23 Acked 1022 Flush 0 Done Sent Done false
193472024-01-24T19:13:57.745Z 24 Acked 1023 Write 10 Done Sent Done false
193482024-01-24T19:13:57.745Z 25 Acked 1024 Flush 0 Done Sent Done false
193492024-01-24T19:13:57.745Z 26 Acked 1025 Write 10 Done Sent Done false
193502024-01-24T19:13:57.745Z 27 Acked 1026 Write 4 Done Sent Done false
193512024-01-24T19:13:57.745Z 28 Acked 1027 Write 8 Done Sent Done false
193522024-01-24T19:13:57.745Z 29 Acked 1028 Write 5 Done Sent Done false
193532024-01-24T19:13:57.745Z 30 Acked 1029 Read 6 Sent Sent Done false
193542024-01-24T19:13:57.745Z STATES DS:0 DS:1 DS:2 TOTAL
193552024-01-24T19:13:57.745Z New 0 0 0 0
193562024-01-24T19:13:57.745Z Sent 1 28 0 29
193572024-01-24T19:13:57.745Z Done 29 2 30 61
193582024-01-24T19:13:57.745Z Skipped 0 0 0 0
193592024-01-24T19:13:57.746Z Error 0 0 0 0
193602024-01-24T19:13:57.746ZLast Flush: 1024 0 1024
193612024-01-24T19:13:57.746ZDownstairs last five completed:
193622024-01-24T19:13:57.746ZUpstairs last five completed: 30 29 28 27 26
193632024-01-24T19:13:57.746ZWrote out file "/tmp/test_repair_verify.data"
193642024-01-24T19:13:57.751Z
193652024-01-24T19:13:57.759ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193662024-01-24T19:13:57.778ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193672024-01-24T19:13:57.778Z 0 000-019 168 168 168 295 295 295 F F F
193682024-01-24T19:13:57.778Z 1 020-039 174 174 174 305 305 305 F F F
193692024-01-24T19:13:57.778Z 2 040-059 180 173 180 315 301 315 F T F <---
193702024-01-24T19:13:57.778Z 3 060-079 180 178 180 315 313 315 F T F <---
193712024-01-24T19:13:57.778Z 4 080-099 177 177 177 310 310 310 F F F
193722024-01-24T19:13:57.778Z 5 100-119 180 176 180 316 307 316 T F T <---
193732024-01-24T19:13:57.778Z 6 120-139 180 176 180 316 307 316 F F F <---
193742024-01-24T19:13:57.778Z 7 140-159 179 179 179 314 314 314 T F T <---
193752024-01-24T19:13:57.784Z 8 160-179 179 179 179 314 314 314 T F T <---
193762024-01-24T19:13:57.785Z 9 180-199 179 179 179 314 314 314 F F F
193772024-01-24T19:13:57.785Z 10 200-219 180 172 180 315 299 315 F F F <---
193782024-01-24T19:13:57.785Z 11 220-239 179 179 179 314 314 314 F F F
193792024-01-24T19:13:57.785Z 12 240-259 179 179 179 314 314 314 F F F
193802024-01-24T19:13:57.785Z 13 260-279 179 179 179 314 314 314 F F F
193812024-01-24T19:13:57.785Z 14 280-299 179 179 179 314 314 314 F F F
193822024-01-24T19:13:57.785Z 15 300-319 180 179 180 316 314 316 F F F <---
193832024-01-24T19:13:57.785Z 16 320-339 178 178 178 313 313 313 F F F
193842024-01-24T19:13:57.785Z 17 340-359 180 179 180 317 314 317 F F F <---
193852024-01-24T19:13:57.785Z 18 360-379 180 178 180 317 313 317 F F F <---
193862024-01-24T19:13:57.785Z 19 380-399 180 179 180 316 314 316 F F F <---
193872024-01-24T19:13:57.785Z 20 400-419 178 178 178 311 311 311 F F F
193882024-01-24T19:13:57.785Z 21 420-439 177 177 177 310 310 310 F F F
193892024-01-24T19:13:57.785Z 22 440-459 180 171 180 316 298 316 F F F <---
193902024-01-24T19:13:57.785Z 23 460-479 178 178 178 312 312 312 F F F
193912024-01-24T19:13:57.785Z 24 480-499 174 174 174 302 302 302 F F F
193922024-01-24T19:13:57.785Z 25 500-519 177 177 177 310 310 310 F F F
193932024-01-24T19:13:57.785Z 26 520-539 180 175 180 316 306 316 T T T <---
193942024-01-24T19:13:57.785Z 27 540-559 180 174 180 316 303 316 F F F <---
193952024-01-24T19:13:57.785Z 28 560-579 174 174 174 304 304 304 F F F
193962024-01-24T19:13:57.785Z 29 580-599 180 177 180 316 310 316 F F F <---
193972024-01-24T19:13:57.785ZMax gen: 180, Max flush: 317
193982024-01-24T19:13:57.785ZError: Difference in extent metadata found!
193992024-01-24T19:13:57.787ZA Difference in extent metadata is expected here
194002024-01-24T19:13:57.787ZOn loop 90
194012024-01-24T19:13:57.787Z
194022024-01-24T19:13:57.792ZVerifying data now
194032024-01-24T19:14:04.109ZNo change after verify
194042024-01-24T19:14:04.110ZLoop: 90 Downstairs dump after verify (and repair):
194052024-01-24T19:14:04.115ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194062024-01-24T19:14:04.115Z 0 000-019 168 168 168 295 295 295 F F F
194072024-01-24T19:14:04.115Z 1 020-039 174 174 174 305 305 305 F F F
194082024-01-24T19:14:04.115Z 2 040-059 180 180 180 315 315 315 F F F
194092024-01-24T19:14:04.115Z 3 060-079 180 180 180 315 315 315 F F F
194102024-01-24T19:14:04.115Z 4 080-099 177 177 177 310 310 310 F F F
194112024-01-24T19:14:04.115Z 5 100-119 181 181 181 318 318 318 F F F
194122024-01-24T19:14:04.116Z 6 120-139 180 180 180 316 316 316 F F F
194132024-01-24T19:14:04.116Z 7 140-159 181 181 181 318 318 318 F F F
194142024-01-24T19:14:04.116Z 8 160-179 181 181 181 318 318 318 F F F
194152024-01-24T19:14:04.116Z 9 180-199 179 179 179 314 314 314 F F F
194162024-01-24T19:14:04.116Z 10 200-219 180 180 180 315 315 315 F F F
194172024-01-24T19:14:04.116Z 11 220-239 179 179 179 314 314 314 F F F
194182024-01-24T19:14:04.116Z 12 240-259 179 179 179 314 314 314 F F F
194192024-01-24T19:14:04.116Z 13 260-279 179 179 179 314 314 314 F F F
194202024-01-24T19:14:04.116Z 14 280-299 179 179 179 314 314 314 F F F
194212024-01-24T19:14:04.116Z 15 300-319 180 180 180 316 316 316 F F F
194222024-01-24T19:14:04.116Z 16 320-339 178 178 178 313 313 313 F F F
194232024-01-24T19:14:04.116Z 17 340-359 180 180 180 317 317 317 F F F
194242024-01-24T19:14:04.116Z 18 360-379 180 180 180 317 317 317 F F F
194252024-01-24T19:14:04.117Z 19 380-399 180 180 180 316 316 316 F F F
194262024-01-24T19:14:04.117Z 20 400-419 178 178 178 311 311 311 F F F
194272024-01-24T19:14:04.117Z 21 420-439 177 177 177 310 310 310 F F F
194282024-01-24T19:14:04.117Z 22 440-459 180 180 180 316 316 316 F F F
194292024-01-24T19:14:04.117Z 23 460-479 178 178 178 312 312 312 F F F
194302024-01-24T19:14:04.117Z 24 480-499 174 174 174 302 302 302 F F F
194312024-01-24T19:14:04.117Z 25 500-519 177 177 177 310 310 310 F F F
194322024-01-24T19:14:04.117Z 26 520-539 181 181 181 318 318 318 F F F
194332024-01-24T19:14:04.117Z 27 540-559 180 180 180 316 316 316 F F F
194342024-01-24T19:14:04.117Z 28 560-579 174 174 174 304 304 304 F F F
194352024-01-24T19:14:04.117Z 29 580-599 180 180 180 316 316 316 F F F
194362024-01-24T19:14:04.117ZMax gen: 181, Max flush: 318
194372024-01-24T19:14:04.117Z
194382024-01-24T19:14:04.117ZBegin loop 91
194392024-01-24T19:14:04.118ZDownstairs to restart: 2
194402024-01-24T19:14:04.138Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815292983Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857}
194412024-01-24T19:14:04.138Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815691486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857}
194422024-01-24T19:14:04.145Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815747832Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857}
194432024-01-24T19:14:04.145Z{"msg":"Crucible 0c19d26c-f4cc-4605-bf61-3a4633842d51 has session id: d5d901c1-565b-4a64-b92d-ee2df70c2006","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815808961Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194442024-01-24T19:14:04.145Z{"msg":"Upstairs opts: Upstairs UUID: 0c19d26c-f4cc-4605-bf61-3a4633842d51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815873403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194452024-01-24T19:14:04.145Z{"msg":"Crucible stats registered with UUID: 0c19d26c-f4cc-4605-bf61-3a4633842d51","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.815953434Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194462024-01-24T19:14:04.145ZCrucible runtime is spawned
194472024-01-24T19:14:04.145Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816260534Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857}
194482024-01-24T19:14:04.145Z{"msg":"0c19d26c-f4cc-4605-bf61-3a4633842d51 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816317211Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194492024-01-24T19:14:04.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.81634308Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194502024-01-24T19:14:04.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816382154Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194512024-01-24T19:14:04.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816424984Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194522024-01-24T19:14:04.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816467165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194532024-01-24T19:14:04.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816506103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194542024-01-24T19:14:04.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816547547Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194552024-01-24T19:14:04.147Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816585257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194562024-01-24T19:14:04.147Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816663611Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194572024-01-24T19:14:04.147Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.816741236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194582024-01-24T19:14:04.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.817013061Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194592024-01-24T19:14:04.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.817098391Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194602024-01-24T19:14:04.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.817127639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194612024-01-24T19:14:04.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818206353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194622024-01-24T19:14:04.147Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818275986Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194632024-01-24T19:14:04.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818303414Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194642024-01-24T19:14:04.147Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818397257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194652024-01-24T19:14:04.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818881229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194662024-01-24T19:14:04.147Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.818964109Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194672024-01-24T19:14:04.147Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.81944737Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194682024-01-24T19:14:04.147Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.819551821Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194692024-01-24T19:14:04.147Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820054531Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194702024-01-24T19:14:04.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820286963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194712024-01-24T19:14:04.147Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820379147Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194722024-01-24T19:14:04.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820464283Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194732024-01-24T19:14:04.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820529841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194742024-01-24T19:14:04.147Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820558122Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194752024-01-24T19:14:04.153Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820581551Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194762024-01-24T19:14:04.153Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820886181Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194772024-01-24T19:14:04.153Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.820993462Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194782024-01-24T19:14:04.153Z{"msg":"[0]R flush_numbers[0..12]: [295, 305, 315, 315, 310, 318, 316, 318, 318, 314, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821136235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194792024-01-24T19:14:04.154Z{"msg":"[0]R generation[0..12]: [168, 174, 180, 180, 177, 181, 180, 181, 181, 179, 180, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821202589Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194802024-01-24T19:14:04.154Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821231247Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194812024-01-24T19:14:04.154Z{"msg":"[1]R flush_numbers[0..12]: [295, 305, 315, 315, 310, 318, 316, 318, 318, 314, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821283931Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194822024-01-24T19:14:04.154Z{"msg":"[1]R generation[0..12]: [168, 174, 180, 180, 177, 181, 180, 181, 181, 179, 180, 179]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
194832024-01-24T19:14:04.154Z,"time":"2024-01-24T19:14:02.821310553Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194842024-01-24T19:14:04.154Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821432919Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194852024-01-24T19:14:04.155Z{"msg":"[2]R flush_numbers[0..12]: [295, 305, 315, 315, 310, 318, 316, 318, 318, 314, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821502637Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194862024-01-24T19:14:04.155Z{"msg":"[2]R generation[0..12]: [168, 174, 180, 180, 177, 181, 180, 181, 181, 179, 180, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821573508Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194872024-01-24T19:14:04.155Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821602194Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194882024-01-24T19:14:04.155Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.82166521Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194892024-01-24T19:14:04.155Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821790144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194902024-01-24T19:14:04.155Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821842231Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194912024-01-24T19:14:04.155Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821875926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"":"downstairs","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194922024-01-24T19:14:04.155Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821899492Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194932024-01-24T19:14:04.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821919227Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"0","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194942024-01-24T19:14:04.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821940952Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"1","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194952024-01-24T19:14:04.156Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.821993393Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"client":"2","":"downstairs","session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194962024-01-24T19:14:04.156Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.822032129Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194972024-01-24T19:14:04.156Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.822089734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
194982024-01-24T19:14:04.156ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194992024-01-24T19:14:04.156Z{"msg":"0c19d26c-f4cc-4605-bf61-3a4633842d51 is now active with session: d5d901c1-565b-4a64-b92d-ee2df70c2006","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.822138497Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
195002024-01-24T19:14:04.156Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.822181938Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857,"session_id":"d5d901c1-565b-4a64-b92d-ee2df70c2006"}
195012024-01-24T19:14:04.156Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:02.822225533Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1857}
195022024-01-24T19:14:04.156ZLoading write count information from file "/tmp/test_repair_verify.data"
195032024-01-24T19:14:04.156ZRun Repair workload
195042024-01-24T19:14:04.156Z01/30 Write block 100 len 24576 data: 8 6 8 12 13 17
195052024-01-24T19:14:04.156Z02/30 Flush
195062024-01-24T19:14:04.156Z03/30 Flush
195072024-01-24T19:14:04.156Z04/30 Write block 288 len 32768 data: 16 15 13 11 11 12 16 14
195082024-01-24T19:14:04.160Z05/30 Write block 589 len 8192 data: 12 11
195092024-01-24T19:14:04.160Z06/30 Read block 33 len 40960
195102024-01-24T19:14:04.160Z07/30 Read block 448 len 20480
195112024-01-24T19:14:04.160Z08/30 Read block 569 len 32768
195122024-01-24T19:14:04.160Z09/30 Read block 331 len 36864
195132024-01-24T19:14:04.164Z10/30 Read block 272 len 16384
195142024-01-24T19:14:04.164Z11/30 Flush
195152024-01-24T19:14:04.167Z12/30 Write block 468 len 20480 data: 12 9 11 14 18
195162024-01-24T19:14:04.167Z13/30 Read block 239 len 8192
195172024-01-24T19:14:04.170Z14/30 Read block 313 len 40960
195182024-01-24T19:14:04.170Z15/30 Write block 188 len 36864 data: 11 11 11 11 12 12 13 11 12
195192024-01-24T19:14:04.173Z16/30 Read block 324 len 16384
195202024-01-24T19:14:04.173Z17/30 Read block 531 len 40960
195212024-01-24T19:14:04.178Z18/30 Read block 446 len 24576
195222024-01-24T19:14:04.178Z19/30 Read block 286 len 16384
195232024-01-24T19:14:04.181Z20/30 Write block 181 len 24576 data: 9 9 9 12 13 13
195242024-01-24T19:14:04.181Z21/30 Write block 3 len 8192 data: 8 8
195252024-01-24T19:14:04.181Z22/30 Read block 182 len 40960
195262024-01-24T19:14:04.184Z23/30 Write block 433 len 36864 data: 16 17 12 11 14 11 12 12 9
195272024-01-24T19:14:04.185Z24/30 Read block 191 len 24576
195282024-01-24T19:14:04.188Z25/30 Write block 568 len 16384 data: 20 21 15 15
195292024-01-24T19:14:04.188Z26/30 Read block 350 len 40960
195302024-01-24T19:14:04.192Z27/30 Read block 6 len 28672
195312024-01-24T19:14:04.192Z28/30 Write block 437 len 8192 data: 15 12
195322024-01-24T19:14:04.192Z29/30 Read block 504 len 12288
195332024-01-24T19:14:04.196Z30/30 Read block 596 len 12288
195342024-01-24T19:14:04.196Z----------------------------------------------------------------
195352024-01-24T19:14:04.196Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:27
195362024-01-24T19:14:04.196ZStates: Active Active Active
195372024-01-24T19:14:04.196ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195382024-01-24T19:14:04.196Z 4 Acked 1003 Write 8 Done Done Done false
195392024-01-24T19:14:04.196Z 5 Acked 1004 Write 2 Done Done Done false
195402024-01-24T19:14:04.197Z 6 Acked 1005 Read 10 Done Done Done false
195412024-01-24T19:14:04.197Z 7 Acked 1006 Read 5 Done Done Done false
195422024-01-24T19:14:04.197Z 8 Acked 1007 Read 8 Done Done Done false
195432024-01-24T19:14:04.197Z 9 Acked 1008 Read 9 Done Done Sent false
195442024-01-24T19:14:04.197Z 10 Acked 1009 Read 4 Done Done Sent false
195452024-01-24T19:14:04.197Z 11 Acked 1010 Flush 0 Done Done Sent false
195462024-01-24T19:14:04.197Z 12 Acked 1011 Write 5 Done Done Sent false
195472024-01-24T19:14:04.197Z 13 Acked 1012 Read 2 Done Done Sent false
195482024-01-24T19:14:04.197Z 14 Acked 1013 Read 10 Done Done Sent false
195492024-01-24T19:14:04.197Z 15 Acked 1014 Write 9 Done Done Sent false
195502024-01-24T19:14:04.197Z 16 Acked 1015 Read 4 Done Done Sent false
195512024-01-24T19:14:04.197Z 17 Acked 1016 Read 10 Done Done Sent false
195522024-01-24T19:14:04.197Z 18 Acked 1017 Read 6 Done Done Sent false
195532024-01-24T19:14:04.197Z 19 Acked 1018 Read 4 Done Done Sent false
195542024-01-24T19:14:04.197Z 20 Acked 1019 Write 6 Done Done Sent false
195552024-01-24T19:14:04.197Z 21 Acked 1020 Write 2 Done Done Sent false
195562024-01-24T19:14:04.197Z 22 Acked 1021 Read 10 Done Done Sent false
195572024-01-24T19:14:04.197Z 23 Acked 1022 Write 9 Done Done Sent false
195582024-01-24T19:14:04.197Z 24 Acked 1023 Read 6 Done Done Sent false
195592024-01-24T19:14:04.197Z 25 Acked 1024 Write 4 Done Done Sent false
195602024-01-24T19:14:04.198Z 26 Acked 1025 Read 10 Done Done Sent false
195612024-01-24T19:14:04.198Z 27 Acked 1026 Read 7 Done Done Sent false
195622024-01-24T19:14:04.198Z 28 Acked 1027 Write 2 Done Done Sent false
195632024-01-24T19:14:04.198Z 29 Acked 1028 Read 3 Done Done Sent false
195642024-01-24T19:14:04.198Z 30 Acked 1029 Read 3 Done Sent Sent false
195652024-01-24T19:14:04.198Z STATES DS:0 DS:1 DS:2 TOTAL
195662024-01-24T19:14:04.198Z New 0 0 0 0
195672024-01-24T19:14:04.198Z Sent 0 1 22 23
195682024-01-24T19:14:04.198Z Done 27 26 5 58
195692024-01-24T19:14:04.198Z Skipped 0 0 0 0
195702024-01-24T19:14:04.199Z Error 0 0 0 0
195712024-01-24T19:14:04.199ZLast Flush: 1010 1010 1002
195722024-01-24T19:14:04.199ZDownstairs last five completed: 1002 1001 1000
195732024-01-24T19:14:04.199ZUpstairs last five completed: 30 29 28 27 26
195742024-01-24T19:14:04.202ZWrote out file "/tmp/test_repair_verify.data"
195752024-01-24T19:14:04.206Z
195762024-01-24T19:14:04.211ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
195772024-01-24T19:14:04.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195782024-01-24T19:14:04.235Z 0 000-019 168 168 168 295 295 295 T T F <---
195792024-01-24T19:14:04.239Z 1 020-039 174 174 174 305 305 305 F F F
195802024-01-24T19:14:04.239Z 2 040-059 180 180 180 315 315 315 F F F
195812024-01-24T19:14:04.239Z 3 060-079 180 180 180 315 315 315 F F F
195822024-01-24T19:14:04.239Z 4 080-099 177 177 177 310 310 310 F F F
195832024-01-24T19:14:04.239Z 5 100-119 182 182 182 319 319 319 F F F
195842024-01-24T19:14:04.240Z 6 120-139 180 180 180 316 316 316 F F F
195852024-01-24T19:14:04.240Z 7 140-159 181 181 181 318 318 318 F F F
195862024-01-24T19:14:04.240Z 8 160-179 181 181 181 318 318 318 F F F
195872024-01-24T19:14:04.240Z 9 180-199 179 179 179 314 314 314 T T F <---
195882024-01-24T19:14:04.240Z 10 200-219 180 180 180 315 315 315 F F F
195892024-01-24T19:14:04.240Z 11 220-239 179 179 179 314 314 314 F F F
195902024-01-24T19:14:04.240Z 12 240-259 179 179 179 314 314 314 F F F
195912024-01-24T19:14:04.240Z 13 260-279 179 179 179 314 314 314 F F F
195922024-01-24T19:14:04.240Z 14 280-299 182 182 179 321 321 314 F F T <---
195932024-01-24T19:14:04.240Z 15 300-319 180 180 180 316 316 316 F F F
195942024-01-24T19:14:04.240Z 16 320-339 178 178 178 313 313 313 F F F
195952024-01-24T19:14:04.240Z 17 340-359 180 180 180 317 317 317 F F F
195962024-01-24T19:14:04.240Z 18 360-379 180 180 180 317 317 317 F F F
195972024-01-24T19:14:04.240Z 19 380-399 180 180 180 316 316 316 F F F
195982024-01-24T19:14:04.240Z 20 400-419 178 178 178 311 311 311 F F F
195992024-01-24T19:14:04.240Z 21 420-439 177 177 177 310 310 310 T T F <---
196002024-01-24T19:14:04.240Z 22 440-459 180 180 180 316 316 316 T T F <---
196012024-01-24T19:14:04.240Z 23 460-479 178 178 178 312 312 312 T T F <---
196022024-01-24T19:14:04.240Z 24 480-499 174 174 174 302 302 302 F F F
196032024-01-24T19:14:04.240Z 25 500-519 177 177 177 310 310 310 F F F
196042024-01-24T19:14:04.240Z 26 520-539 181 181 181 318 318 318 F F F
196052024-01-24T19:14:04.240Z 27 540-559 180 180 180 316 316 316 F F F
196062024-01-24T19:14:04.240Z 28 560-579 174 174 174 304 304 304 T T F <---
196072024-01-24T19:14:04.241Z 29 580-599 182 182 180 321 321 316 F F T <---
196082024-01-24T19:14:04.243ZMax gen: 182, Max flush: 321
196092024-01-24T19:14:04.243ZError: Difference in extent metadata found!
196102024-01-24T19:14:04.247ZA Difference in extent metadata is expected here
196112024-01-24T19:14:04.247ZOn loop 91
196122024-01-24T19:14:04.247Z
196132024-01-24T19:14:04.247ZVerifying data now
196142024-01-24T19:14:10.026ZNo change after verify
196152024-01-24T19:14:10.026ZLoop: 91 Downstairs dump after verify (and repair):
196162024-01-24T19:14:10.050ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196172024-01-24T19:14:10.050Z 0 000-019 183 183 183 322 322 322 F F F
196182024-01-24T19:14:10.050Z 1 020-039 174 174 174 305 305 305 F F F
196192024-01-24T19:14:10.050Z 2 040-059 180 180 180 315 315 315 F F F
196202024-01-24T19:14:10.056Z 3 060-079 180 180 180 315 315 315 F F F
196212024-01-24T19:14:10.056Z 4 080-099 177 177 177 310 310 310 F F F
196222024-01-24T19:14:10.057Z 5 100-119 182 182 182 319 319 319 F F F
196232024-01-24T19:14:10.057Z 6 120-139 180 180 180 316 316 316 F F F
196242024-01-24T19:14:10.057Z 7 140-159 181 181 181 318 318 318 F F F
196252024-01-24T19:14:10.057Z 8 160-179 181 181 181 318 318 318 F F F
196262024-01-24T19:14:10.057Z 9 180-199 183 183 183 322 322 322 F F F
196272024-01-24T19:14:10.057Z 10 200-219 180 180 180 315 315 315 F F F
196282024-01-24T19:14:10.057Z 11 220-239 179 179 179 314 314 314 F F F
196292024-01-24T19:14:10.057Z 12 240-259 179 179 179 314 314 314 F F F
196302024-01-24T19:14:10.057Z 13 260-279 179 179 179 314 314 314 F F F
196312024-01-24T19:14:10.057Z 14 280-299 182 182 182 321 321 321 F F F
196322024-01-24T19:14:10.057Z 15 300-319 180 180 180 316 316 316 F F F
196332024-01-24T19:14:10.057Z 16 320-339 178 178 178 313 313 313 F F F
196342024-01-24T19:14:10.058Z 17 340-359 180 180 180 317 317 317 F F F
196352024-01-24T19:14:10.058Z 18 360-379 180 180 180 317 317 317 F F F
196362024-01-24T19:14:10.058Z 19 380-399 180 180 180 316 316 316 F F F
196372024-01-24T19:14:10.058Z 20 400-419 178 178 178 311 311 311 F F F
196382024-01-24T19:14:10.058Z 21 420-439 183 183 183 322 322 322 F F F
196392024-01-24T19:14:10.058Z 22 440-459 183 183 183 322 322 322 F F F
196402024-01-24T19:14:10.058Z 23 460-479 183 183 183 322 322 322 F F F
196412024-01-24T19:14:10.058Z 24 480-499 174 174 174 302 302 302 F F F
196422024-01-24T19:14:10.058Z 25 500-519 177 177 177 310 310 310 F F F
196432024-01-24T19:14:10.058Z 26 520-539 181 181 181 318 318 318 F F F
196442024-01-24T19:14:10.058Z 27 540-559 180 180 180 316 316 316 F F F
196452024-01-24T19:14:10.058Z 28 560-579 183 183 183 322 322 322 F F F
196462024-01-24T19:14:10.059Z 29 580-599 182 182 182 321 321 321 F F F
196472024-01-24T19:14:10.059ZMax gen: 183, Max flush: 322
196482024-01-24T19:14:10.059Z
196492024-01-24T19:14:10.059ZBegin loop 92
196502024-01-24T19:14:10.060ZDownstairs to restart: 1
196512024-01-24T19:14:10.083Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762034888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865}
196522024-01-24T19:14:10.083Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762475667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865}
196532024-01-24T19:14:10.093Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762595495Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865}
196542024-01-24T19:14:10.093Z{"msg":"Crucible 0e96a88b-e05f-446e-b1c4-da43dda4bfa6 has session id: 73174ece-6a3c-49d1-b09c-f3564026bb3d","v":0,"name":"crucible","level":30Crucible runtime is spawned
196552024-01-24T19:14:10.093Z,"time":"2024-01-24T19:14:08.762675254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196562024-01-24T19:14:10.093Z{"msg":"Upstairs opts: Upstairs UUID: 0e96a88b-e05f-446e-b1c4-da43dda4bfa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762847528Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196572024-01-24T19:14:10.093Z{"msg":"Crucible stats registered with UUID: 0e96a88b-e05f-446e-b1c4-da43dda4bfa6","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762924327Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196582024-01-24T19:14:10.093Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.762950021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865}
196592024-01-24T19:14:10.093Z{"msg":"0e96a88b-e05f-446e-b1c4-da43dda4bfa6 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763170032Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196602024-01-24T19:14:10.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76323001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196612024-01-24T19:14:10.094Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763258371Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196622024-01-24T19:14:10.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763280905Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196632024-01-24T19:14:10.094Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763303998Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196642024-01-24T19:14:10.094Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763352751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196652024-01-24T19:14:10.095Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76339186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196662024-01-24T19:14:10.095Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763448482Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196672024-01-24T19:14:10.095Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763481626Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196682024-01-24T19:14:10.095Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763608248Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196692024-01-24T19:14:10.095Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763692377Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196702024-01-24T19:14:10.096Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763765891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196712024-01-24T19:14:10.096Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.763861183Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196722024-01-24T19:14:10.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.765119192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196732024-01-24T19:14:10.096Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.765197042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196742024-01-24T19:14:10.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.765303089Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196752024-01-24T19:14:10.096Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.765344731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196762024-01-24T19:14:10.096Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.766242292Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196772024-01-24T19:14:10.096Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.766456079Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196782024-01-24T19:14:10.096Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.766672968Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196792024-01-24T19:14:10.096Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.766765049Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196802024-01-24T19:14:10.097Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767140324Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196812024-01-24T19:14:10.097Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767208847Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196822024-01-24T19:14:10.097Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767245926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196832024-01-24T19:14:10.097Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767404573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196842024-01-24T19:14:10.098Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767466514Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196852024-01-24T19:14:10.098Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767544262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196862024-01-24T19:14:10.098Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.767765863Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196872024-01-24T19:14:10.098Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768512945Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196882024-01-24T19:14:10.098Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768609993Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196892024-01-24T19:14:10.098Z{"msg":"[0]R flush_numbers[0..12]: [322, 305, 315, 315, 310, 319, 316, 318, 318, 322, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768686541Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196902024-01-24T19:14:10.098Z{"msg":"[0]R generation[0..12]: [183, 174, 180, 180, 177, 182, 180, 181, 181, 183, 180, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76873062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196912024-01-24T19:14:10.098Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768797024Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196922024-01-24T19:14:10.098Z{"msg":"[1]R flush_numbers[0..12]: [322, 305, 315, 315, 310, 319, 316, 318, 318, 322, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76884427Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196932024-01-24T19:14:10.098ZWait for a query_work_queue command to finish before sending IO
196942024-01-24T19:14:10.098Z{"msg":"[1]R generation[0..12]: [183, 174, 180, 180, 177, 182, 180, 181, 181, 183, 180, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768909773Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196952024-01-24T19:14:10.104Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.768955868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196962024-01-24T19:14:10.104Z{"msg":"[2]R flush_numbers[0..12]: [322, 305, 315, 315, 310, 319, 316, 318, 318, 322, 315, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76912141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196972024-01-24T19:14:10.104Z{"msg":"[2]R generation[0..12]: [183, 174, 180, 180, 177, 182, 180, 181, 181, 183, 180, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769173665Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196982024-01-24T19:14:10.104Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769238057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
196992024-01-24T19:14:10.105Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769282163Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197002024-01-24T19:14:10.105Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76933936Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197012024-01-24T19:14:10.105Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769381274Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197022024-01-24T19:14:10.105Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769440156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"":"downstairs","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197032024-01-24T19:14:10.106Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769483558Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197042024-01-24T19:14:10.106ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197052024-01-24T19:14:10.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769544907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"0","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197062024-01-24T19:14:10.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769588612Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"1","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197072024-01-24T19:14:10.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769642891Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"client":"2","":"downstairs","session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197082024-01-24T19:14:10.106Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769686011Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197092024-01-24T19:14:10.106Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769740824Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197102024-01-24T19:14:10.106Z{"msg":"0e96a88b-e05f-446e-b1c4-da43dda4bfa6 is now active with session: 73174ece-6a3c-49d1-b09c-f3564026bb3d","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769782757Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197112024-01-24T19:14:10.106Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.769835402Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865,"session_id":"73174ece-6a3c-49d1-b09c-f3564026bb3d"}
197122024-01-24T19:14:10.106Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:08.76987507Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1865}
197132024-01-24T19:14:10.106ZLoading write count information from file "/tmp/test_repair_verify.data"
197142024-01-24T19:14:10.106ZRun Repair workload
197152024-01-24T19:14:10.106Z01/30 Write block 31 len 16384 data: 18 16 16 16
197162024-01-24T19:14:10.106Z02/30 Write block 382 len 36864 data: 14 15 12 10 10 11 13 11 11
197172024-01-24T19:14:10.106Z03/30 Read block 500 len 28672
197182024-01-24T19:14:10.106Z04/30 Read block 214 len 40960
197192024-01-24T19:14:10.106Z05/30 Read block 316 len 16384
197202024-01-24T19:14:10.107Z06/30 Flush
197212024-01-24T19:14:10.123Z07/30 Write block 136 len 20480 data: 18 17 15 15 13
197222024-01-24T19:14:10.123Z08/30 Read block 200 len 12288
197232024-01-24T19:14:10.124Z09/30 Read block 234 len 8192
197242024-01-24T19:14:10.124Z10/30 Write block 314 len 36864 data: 18 18 18 17 16 15 14 13 13
197252024-01-24T19:14:10.124Z11/30 Read block 261 len 40960
197262024-01-24T19:14:10.124Z12/30 Write block 26 len 4096 data: 12
197272024-01-24T19:14:10.124Z13/30 Flush
197282024-01-24T19:14:10.124Z14/30 Write block 554 len 24576 data: 13 12 13 15 15 16
197292024-01-24T19:14:10.124Z15/30 Flush
197302024-01-24T19:14:10.124Z16/30 Write block 161 len 40960 data: 21 20 21 17 22 19 20 20 17 17
197312024-01-24T19:14:10.130Z17/30 Write block 200 len 20480 data: 9 9 7 8 10
197322024-01-24T19:14:10.130Z18/30 Write block 385 len 36864 data: 11 11 12 14 12 12 10 10 12
197332024-01-24T19:14:10.130Z19/30 Read block 47 len 20480
197342024-01-24T19:14:10.131Z20/30 Read block 75 len 8192
197352024-01-24T19:14:10.131Z21/30 Read block 406 len 20480
197362024-01-24T19:14:10.137Z22/30 Write block 271 len 40960 data: 14 18 19 15 15 13 15 15 18 18
197372024-01-24T19:14:10.137Z23/30 Read block 315 len 4096
197382024-01-24T19:14:10.137Z24/30 Read block 458 len 24576
197392024-01-24T19:14:10.137Z25/30 Read block 384 len 36864
197402024-01-24T19:14:10.144Z26/30 Write block 40 len 16384 data: 9 12 11 12
197412024-01-24T19:14:10.145Z27/30 Write block 1 len 20480 data: 6 7 9 9 9
197422024-01-24T19:14:10.145Z28/30 Write block 96 len 28672 data: 10 12 11 9 9 7 9
197432024-01-24T19:14:10.145Z29/30 Write block 88 len 8192 data: 9 8
197442024-01-24T19:14:10.145Z30/30 Write block 343 len 8192 data: 15 15
197452024-01-24T19:14:10.145Z----------------------------------------------------------------
197462024-01-24T19:14:10.145Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
197472024-01-24T19:14:10.145ZStates: Active Active Active
197482024-01-24T19:14:10.145ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197492024-01-24T19:14:10.145Z 1 Acked 1000 Write 4 Done Done Done false
197502024-01-24T19:14:10.145Z 2 Acked 1001 Write 9 Done Done Done false
197512024-01-24T19:14:10.145Z 3 Acked 1002 Read 7 Done Done Done false
197522024-01-24T19:14:10.145Z 4 Acked 1003 Read 10 Done Sent Done false
197532024-01-24T19:14:10.146Z 5 Acked 1004 Read 4 Done Sent Done false
197542024-01-24T19:14:10.146Z 6 Acked 1005 Flush 0 Done Sent Done false
197552024-01-24T19:14:10.146Z 7 Acked 1006 Write 5 Done Sent Done false
197562024-01-24T19:14:10.146Z 8 Acked 1007 Read 3 Done Sent Done false
197572024-01-24T19:14:10.147Z 9 Acked 1008 Read 2 Done Sent Done false
197582024-01-24T19:14:10.147Z 10 Acked 1009 Write 9 Done Sent Done false
197592024-01-24T19:14:10.147Z 11 Acked 1010 Read 10 Done Sent Done false
197602024-01-24T19:14:10.147Z 12 Acked 1011 Write 1 Done Sent Done false
197612024-01-24T19:14:10.147Z 13 Acked 1012 Flush 0 Done Sent Done false
197622024-01-24T19:14:10.147Z 14 Acked 1013 Write 6 Done Sent Done false
197632024-01-24T19:14:10.147Z 15 Acked 1014 Flush 0 Done Sent Done false
197642024-01-24T19:14:10.147Z 16 Acked 1015 Write 10 Done Sent Done false
197652024-01-24T19:14:10.147Z 17 Acked 1016 Write 5 Done Sent Done false
197662024-01-24T19:14:10.147Z 18 Acked 1017 Write 9 Done Sent Done false
197672024-01-24T19:14:10.147Z 19 Acked 1018 Read 5 Done Sent Done false
197682024-01-24T19:14:10.147Z 20 Acked 1019 Read 2 Done Sent Done false
197692024-01-24T19:14:10.148Z 21 Acked 1020 Read 5 Done Sent Done false
197702024-01-24T19:14:10.148Z 22 Acked 1021 Write 10 Done Sent Done false
197712024-01-24T19:14:10.148Z 23 Acked 1022 Read 1 Done Sent Done false
197722024-01-24T19:14:10.148Z 24 Acked 1023 Read 6 Done Sent Done false
197732024-01-24T19:14:10.148Z 25 Acked 1024 Read 9 Done Sent Done false
197742024-01-24T19:14:10.148Z 26 Acked 1025 Write 4 Done Sent Done false
197752024-01-24T19:14:10.148Z 27 Acked 1026 Write 5 Done Sent Done false
197762024-01-24T19:14:10.148Z 28 Acked 1027 Write 7 Done Sent Done false
197772024-01-24T19:14:10.148Z 29 Acked 1028 Write 2 Done Sent Done false
197782024-01-24T19:14:10.150Z 30 Acked 1029 Write 2 Sent Sent Sent false
197792024-01-24T19:14:10.150Z STATES DS:0 DS:1 DS:2 TOTAL
197802024-01-24T19:14:10.150Z New 0 0 0 0
197812024-01-24T19:14:10.150Z Sent 1 27 1 29
197822024-01-24T19:14:10.150Z Done 29 3 29 61
197832024-01-24T19:14:10.150Z Skipped 0 0 0 0
197842024-01-24T19:14:10.150Z Error 0 0 0 0
197852024-01-24T19:14:10.150ZLast Flush: 1014 0 1014
197862024-01-24T19:14:10.151ZDownstairs last five completed:
197872024-01-24T19:14:10.151ZUpstairs last five completed: 30 29 28 27 26
197882024-01-24T19:14:10.151ZWrote out file "/tmp/test_repair_verify.data"
197892024-01-24T19:14:10.156Z
197902024-01-24T19:14:10.156ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197912024-01-24T19:14:10.197ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197922024-01-24T19:14:10.197Z 0 000-019 183 183 183 322 322 322 T F T <---
197932024-01-24T19:14:10.198Z 1 020-039 184 174 184 324 305 324 F T F <---
197942024-01-24T19:14:10.198Z 2 040-059 180 180 180 315 315 315 T F T <---
197952024-01-24T19:14:10.198Z 3 060-079 180 180 180 315 315 315 F F F
197962024-01-24T19:14:10.198Z 4 080-099 177 177 177 310 310 310 T F T <---
197972024-01-24T19:14:10.198Z 5 100-119 182 182 182 319 319 319 T F T <---
197982024-01-24T19:14:10.198Z 6 120-139 184 180 184 324 316 324 F F F <---
197992024-01-24T19:14:10.199Z 7 140-159 184 181 184 324 318 324 F F F <---
198002024-01-24T19:14:10.210Z 8 160-179 181 181 181 318 318 318 T F T <---
198012024-01-24T19:14:10.211Z 9 180-199 183 183 183 322 322 322 F F F
198022024-01-24T19:14:10.211Z 10 200-219 180 180 180 315 315 315 T F T <---
198032024-01-24T19:14:10.223Z 11 220-239 179 179 179 314 314 314 F F F
198042024-01-24T19:14:10.224Z 12 240-259 179 179 179 314 314 314 F F F
198052024-01-24T19:14:10.224Z 13 260-279 179 179 179 314 314 314 T F T <---
198062024-01-24T19:14:10.224Z 14 280-299 182 182 182 321 321 321 T F T <---
198072024-01-24T19:14:10.224Z 15 300-319 184 180 184 324 316 324 F F F <---
198082024-01-24T19:14:10.224Z 16 320-339 184 178 184 324 313 324 F F F <---
198092024-01-24T19:14:10.224Z 17 340-359 180 180 180 317 317 317 T F T <---
198102024-01-24T19:14:10.224Z 18 360-379 180 180 180 317 317 317 F F F
198112024-01-24T19:14:10.225Z 19 380-399 184 180 184 323 316 323 T T T <---
198122024-01-24T19:14:10.225Z 20 400-419 178 178 178 311 311 311 F F F
198132024-01-24T19:14:10.225Z 21 420-439 183 183 183 322 322 322 F F F
198142024-01-24T19:14:10.225Z 22 440-459 183 183 183 322 322 322 F F F
198152024-01-24T19:14:10.225Z 23 460-479 183 183 183 322 322 322 F F F
198162024-01-24T19:14:10.225Z 24 480-499 174 174 174 302 302 302 F F F
198172024-01-24T19:14:10.225Z 25 500-519 177 177 177 310 310 310 F F F
198182024-01-24T19:14:10.226Z 26 520-539 181 181 181 318 318 318 F F F
198192024-01-24T19:14:10.226Z 27 540-559 184 180 184 325 316 325 F F F <---
198202024-01-24T19:14:10.226Z 28 560-579 183 183 183 322 322 322 F F F
198212024-01-24T19:14:10.226Z 29 580-599 182 182 182 321 321 321 F F F
198222024-01-24T19:14:10.226ZMax gen: 184, Max flush: 325
198232024-01-24T19:14:10.226ZError: Difference in extent metadata found!
198242024-01-24T19:14:10.226ZA Difference in extent metadata is expected here
198252024-01-24T19:14:10.226ZOn loop 92
198262024-01-24T19:14:10.226Z
198272024-01-24T19:14:10.226ZVerifying data now
198282024-01-24T19:14:16.579ZNo change after verify
198292024-01-24T19:14:16.579ZLoop: 92 Downstairs dump after verify (and repair):
198302024-01-24T19:14:16.602ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198312024-01-24T19:14:16.602Z 0 000-019 185 185 185 326 326 326 F F F
198322024-01-24T19:14:16.605Z 1 020-039 184 184 184 324 324 324 F F F
198332024-01-24T19:14:16.605Z 2 040-059 185 185 185 326 326 326 F F F
198342024-01-24T19:14:16.606Z 3 060-079 180 180 180 315 315 315 F F F
198352024-01-24T19:14:16.606Z 4 080-099 185 185 185 326 326 326 F F F
198362024-01-24T19:14:16.606Z 5 100-119 185 185 185 326 326 326 F F F
198372024-01-24T19:14:16.606Z 6 120-139 184 184 184 324 324 324 F F F
198382024-01-24T19:14:16.606Z 7 140-159 184 184 184 324 324 324 F F F
198392024-01-24T19:14:16.606Z 8 160-179 185 185 185 326 326 326 F F F
198402024-01-24T19:14:16.606Z 9 180-199 183 183 183 322 322 322 F F F
198412024-01-24T19:14:16.606Z 10 200-219 185 185 185 326 326 326 F F F
198422024-01-24T19:14:16.606Z 11 220-239 179 179 179 314 314 314 F F F
198432024-01-24T19:14:16.606Z 12 240-259 179 179 179 314 314 314 F F F
198442024-01-24T19:14:16.606Z 13 260-279 185 185 185 326 326 326 F F F
198452024-01-24T19:14:16.606Z 14 280-299 185 185 185 326 326 326 F F F
198462024-01-24T19:14:16.606Z 15 300-319 184 184 184 324 324 324 F F F
198472024-01-24T19:14:16.606Z 16 320-339 184 184 184 324 324 324 F F F
198482024-01-24T19:14:16.606Z 17 340-359 185 185 185 326 326 326 F F F
198492024-01-24T19:14:16.606Z 18 360-379 180 180 180 317 317 317 F F F
198502024-01-24T19:14:16.606Z 19 380-399 185 185 185 326 326 326 F F F
198512024-01-24T19:14:16.606Z 20 400-419 178 178 178 311 311 311 F F F
198522024-01-24T19:14:16.606Z 21 420-439 183 183 183 322 322 322 F F F
198532024-01-24T19:14:16.606Z 22 440-459 183 183 183 322 322 322 F F F
198542024-01-24T19:14:16.606Z 23 460-479 183 183 183 322 322 322 F F F
198552024-01-24T19:14:16.606Z 24 480-499 174 174 174 302 302 302 F F F
198562024-01-24T19:14:16.606Z 25 500-519 177 177 177 310 310 310 F F F
198572024-01-24T19:14:16.607Z 26 520-539 181 181 181 318 318 318 F F F
198582024-01-24T19:14:16.607Z 27 540-559 184 184 184 325 325 325 F F F
198592024-01-24T19:14:16.607Z 28 560-579 183 183 183 322 322 322 F F F
198602024-01-24T19:14:16.607Z 29 580-599 182 182 182 321 321 321 F F F
198612024-01-24T19:14:16.607ZMax gen: 185, Max flush: 326
198622024-01-24T19:14:16.611Z
198632024-01-24T19:14:16.611ZBegin loop 93
198642024-01-24T19:14:16.612ZDownstairs to restart: 1
198652024-01-24T19:14:16.661Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.3138282Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873}
198662024-01-24T19:14:16.669Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314251339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873}
198672024-01-24T19:14:16.669Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.31432555Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873}
198682024-01-24T19:14:16.670Z{"msg":"Crucible 9a9527b4-7b6d-4dda-bae9-bb0e73c649fc has session id: de1d9f08-f73e-4adf-a61f-2ee876b4e060","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314365662Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198692024-01-24T19:14:16.670Z{"msg":"Upstairs opts: Upstairs UUID: 9a9527b4-7b6d-4dda-bae9-bb0e73c649fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314416628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198702024-01-24T19:14:16.671Z{"msg":"Crucible runtime is spawned
198712024-01-24T19:14:16.671ZCrucible stats registered with UUID: 9a9527b4-7b6d-4dda-bae9-bb0e73c649fc","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314647484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198722024-01-24T19:14:16.671Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314745165Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873}
198732024-01-24T19:14:16.671Z{"msg":"9a9527b4-7b6d-4dda-bae9-bb0e73c649fc active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314887315Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198742024-01-24T19:14:16.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314957339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198752024-01-24T19:14:16.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.314986121Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198762024-01-24T19:14:16.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315078333Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198772024-01-24T19:14:16.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315177622Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198782024-01-24T19:14:16.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315304499Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198792024-01-24T19:14:16.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315412185Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198802024-01-24T19:14:16.673Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315490196Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198812024-01-24T19:14:16.673Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315570638Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198822024-01-24T19:14:16.673Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315617229Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198832024-01-24T19:14:16.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315754457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198842024-01-24T19:14:16.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315802449Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198852024-01-24T19:14:16.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.315875262Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198862024-01-24T19:14:16.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.316910711Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198872024-01-24T19:14:16.673Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.316974239Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198882024-01-24T19:14:16.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.317052323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198892024-01-24T19:14:16.673Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.317170177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198902024-01-24T19:14:16.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.317353619Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198912024-01-24T19:14:16.673Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.317400539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198922024-01-24T19:14:16.673Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.318241673Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198932024-01-24T19:14:16.673Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.318357783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198942024-01-24T19:14:16.673Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.318477679Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198952024-01-24T19:14:16.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.31925843Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198962024-01-24T19:14:16.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319358566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198972024-01-24T19:14:16.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319413518Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198982024-01-24T19:14:16.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319483432Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
198992024-01-24T19:14:16.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319528263Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199002024-01-24T19:14:16.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319585268Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199012024-01-24T19:14:16.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319626108Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199022024-01-24T19:14:16.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319693658Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2"Wait for a query_work_queue command to finish before sending IO
199032024-01-24T19:14:16.674Z,"":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199042024-01-24T19:14:16.674Z{"msg":"[0]R flush_numbers[0..12]: [326, 324, 326, 315, 326, 326, 324, 324, 326, 322, 326, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319744947Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199052024-01-24T19:14:16.674Z{"msg":"[0]R generation[0..12]: [185, 184, 185, 180, 185, 185, 184, 184, 185, 183, 185, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319803372Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199062024-01-24T19:14:16.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319849475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199072024-01-24T19:14:16.674Z{"msg":"[1]R flush_numbers[0..12]: [326, 324, 326, 315, 326, 326, 324, 324, 326, 322, 326, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319907987Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199082024-01-24T19:14:16.674Z{"msg":"[1]R generation[0..12]: [185, 184, 185, 180, 185, 185, 184, 184, 185, 183, 185, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.319953538Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199092024-01-24T19:14:16.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320019759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199102024-01-24T19:14:16.675Z{"msg":"[2]R flush_numbers[0..12]: [326, 324, 326, 315, 326, 326, 324, 324, 326, 322, 326, 314]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320064369Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199112024-01-24T19:14:16.675Z{"msg":"[2]R generation[0..12]: [185, 184, 185, 180, 185, 185, 184, 184, 185, 183, 185, 179]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320131452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199122024-01-24T19:14:16.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.32017808Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199132024-01-24T19:14:16.675Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320235703Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199142024-01-24T19:14:16.675Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320279052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199152024-01-24T19:14:16.675Z{"msg":"Next flush: 327","v":0,"nameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199162024-01-24T19:14:16.675Z":"crucible","level":30,"time":"2024-01-24T19:14:15.320348842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199172024-01-24T19:14:16.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320392911Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"":"downstairs","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199182024-01-24T19:14:16.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320448954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199192024-01-24T19:14:16.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320488789Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"0","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199202024-01-24T19:14:16.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320545247Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"1","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199212024-01-24T19:14:16.676Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320588697Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"client":"2","":"downstairs","session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199222024-01-24T19:14:16.676Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320644112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199232024-01-24T19:14:16.676Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320684591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199242024-01-24T19:14:16.676Z{"msg":"9a9527b4-7b6d-4dda-bae9-bb0e73c649fc is now active with session: de1d9f08-f73e-4adf-a61f-2ee876b4e060","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320741438Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199252024-01-24T19:14:16.676Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320781787Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873,"session_id":"de1d9f08-f73e-4adf-a61f-2ee876b4e060"}
199262024-01-24T19:14:16.676Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:15.320836566Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1873}
199272024-01-24T19:14:16.676ZLoading write count information from file "/tmp/test_repair_verify.data"
199282024-01-24T19:14:16.676ZRun Repair workload
199292024-01-24T19:14:16.676Z01/30 Write block 107 len 20480 data: 20 20 21 20 18
199302024-01-24T19:14:16.676Z02/30 Write block 436 len 20480 data: 12 16 13 13 13
199312024-01-24T19:14:16.676Z03/30 Flush
199322024-01-24T19:14:16.676Z04/30 Write block 226 len 12288 data: 12 12 11
199332024-01-24T19:14:16.676Z05/30 Read block 517 len 4096
199342024-01-24T19:14:16.676Z06/30 Flush
199352024-01-24T19:14:16.677Z07/30 Write block 174 len 4096 data: 19
199362024-01-24T19:14:16.677Z08/30 Write block 263 len 8192 data: 11 14
199372024-01-24T19:14:16.677Z09/30 Read block 307 len 4096
199382024-01-24T19:14:16.677Z10/30 Flush
199392024-01-24T19:14:16.678Z11/30 Flush
199402024-01-24T19:14:16.678Z12/30 Write block 288 len 24576 data: 17 16 14 12 12 13
199412024-01-24T19:14:16.678Z13/30 Read block 402 len 32768
199422024-01-24T19:14:16.678Z14/30 Read block 430 len 36864
199432024-01-24T19:14:16.682Z15/30 Write block 268 len 32768 data: 12 13 14 15 19 20 16 16
199442024-01-24T19:14:16.682Z16/30 Write block 133 len 36864 data: 13 17 17 19 18 16 16 14 10
199452024-01-24T19:14:16.682Z17/30 Read block 61 len 32768
199462024-01-24T19:14:16.682Z18/30 Write block 6 len 36864 data: 11 12 14 11 11 10 8 8 8
199472024-01-24T19:14:16.682Z19/30 Write block 381 len 12288 data: 16 15 16
199482024-01-24T19:14:16.682Z20/30 Write block 201 len 20480 data: 10 8 9 11 10
199492024-01-24T19:14:16.682Z21/30 Read block 195 len 20480
199502024-01-24T19:14:16.682Z22/30 Write block 105 len 36864 data: 18 18 21 21 22 21 19 14 12
199512024-01-24T19:14:16.682Z23/30 Write block 417 len 8192 data: 11 13
199522024-01-24T19:14:16.682Z24/30 Flush
199532024-01-24T19:14:16.682Z25/30 Write block 554 len 16384 data: 14 13 14 16
199542024-01-24T19:14:16.694Z26/30 Write block 406 len 32768 data: 14 13 15 9 11 13 12 11
199552024-01-24T19:14:16.694Z27/30 Write block 456 len 12288 data: 14 12 13
199562024-01-24T19:14:16.694Z28/30 Read block 105 len 4096
199572024-01-24T19:14:16.694Z29/30 Read block 360 len 36864
199582024-01-24T19:14:16.694Z30/30 Read block 421 len 20480
199592024-01-24T19:14:16.694Z----------------------------------------------------------------
199602024-01-24T19:14:16.694Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:27
199612024-01-24T19:14:16.695ZStates: Active Active Active
199622024-01-24T19:14:16.695ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199632024-01-24T19:14:16.695Z 4 Acked 1003 Write 3 Done Sent Done false
199642024-01-24T19:14:16.695Z 5 Acked 1004 Read 1 Done Sent Done false
199652024-01-24T19:14:16.695Z 6 Acked 1005 Flush 0 Done Sent Done false
199662024-01-24T19:14:16.696Z 7 Acked 1006 Write 1 Done Sent Done false
199672024-01-24T19:14:16.696Z 8 Acked 1007 Write 2 Done Sent Done false
199682024-01-24T19:14:16.696Z 9 Acked 1008 Read 1 Done Sent Done false
199692024-01-24T19:14:16.696Z 10 Acked 1009 Flush 0 Done Sent Done false
199702024-01-24T19:14:16.696Z 11 Acked 1010 Flush 0 Done Sent Done false
199712024-01-24T19:14:16.696Z 12 Acked 1011 Write 6 Done Sent Done false
199722024-01-24T19:14:16.696Z 13 Acked 1012 Read 8 Done Sent Done false
199732024-01-24T19:14:16.696Z 14 Acked 1013 Read 9 Done Sent Done false
199742024-01-24T19:14:16.696Z 15 Acked 1014 Write 8 Done Sent Done false
199752024-01-24T19:14:16.696Z 16 Acked 1015 Write 9 Done Sent Done false
199762024-01-24T19:14:16.696Z 17 Acked 1016 Read 8 Done Sent Done false
199772024-01-24T19:14:16.696Z 18 Acked 1017 Write 9 Done Sent Done false
199782024-01-24T19:14:16.696Z 19 Acked 1018 Write 3 Done Sent Done false
199792024-01-24T19:14:16.696Z 20 Acked 1019 Write 5 Done Sent Done false
199802024-01-24T19:14:16.696Z 21 Acked 1020 Read 5 Done Sent Done false
199812024-01-24T19:14:16.697Z 22 Acked 1021 Write 9 Done Sent Done false
199822024-01-24T19:14:16.697Z 23 Acked 1022 Write 2 Done Sent Done false
199832024-01-24T19:14:16.697Z 24 Acked 1023 Flush 0 Done Sent Done false
199842024-01-24T19:14:16.697Z 25 Acked 1024 Write 4 Done Sent Done false
199852024-01-24T19:14:16.697Z 26 Acked 1025 Write 8 Done Sent Done false
199862024-01-24T19:14:16.697Z 27 Acked 1026 Write 3 Done Sent Done false
199872024-01-24T19:14:16.697Z 28 Acked 1027 Read 1 Done Sent Done false
199882024-01-24T19:14:16.697Z 29 Acked 1028 Read 9 Done Sent Done false
199892024-01-24T19:14:16.697Z 30 Acked 1029 Read 5 Sent Sent Done false
199902024-01-24T19:14:16.697Z STATES DS:0 DS:1 DS:2 TOTAL
199912024-01-24T19:14:16.697Z New 0 0 0 0
199922024-01-24T19:14:16.697Z Sent 1 27 0 28
199932024-01-24T19:14:16.697Z Done 26 0 27 53
199942024-01-24T19:14:16.697Z Skipped 0 0 0 0
199952024-01-24T19:14:16.698Z Error 0 0 0 0
199962024-01-24T19:14:16.698ZLast Flush: 1023 1002 1023
199972024-01-24T19:14:16.698ZDownstairs last five completed: 1002 1001 1000
199982024-01-24T19:14:16.698ZUpstairs last five completed: 30 29 28 27 26
199992024-01-24T19:14:16.698ZWrote out file "/tmp/test_repair_verify.data"
200002024-01-24T19:14:16.702Z
200012024-01-24T19:14:16.705ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200022024-01-24T19:14:16.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200032024-01-24T19:14:16.729Z 0 000-019 186 185 186 331 326 331 F F F <---
200042024-01-24T19:14:16.730Z 1 020-039 184 184 184 324 324 324 F F F
200052024-01-24T19:14:16.730Z 2 040-059 185 185 185 326 326 326 F F F
200062024-01-24T19:14:16.730Z 3 060-079 180 180 180 315 315 315 F F F
200072024-01-24T19:14:16.730Z 4 080-099 185 185 185 326 326 326 F F F
200082024-01-24T19:14:16.730Z 5 100-119 186 186 186 331 327 331 F F F <---
200092024-01-24T19:14:16.730Z 6 120-139 186 184 186 331 324 331 F F F <---
200102024-01-24T19:14:16.730Z 7 140-159 186 184 186 331 324 331 F F F <---
200112024-01-24T19:14:16.730Z 8 160-179 186 185 186 329 326 329 F F F <---
200122024-01-24T19:14:16.730Z 9 180-199 183 183 183 322 322 322 F F F
200132024-01-24T19:14:16.730Z 10 200-219 186 185 186 331 326 331 F F F <---
200142024-01-24T19:14:16.730Z 11 220-239 186 179 186 328 314 328 F F F <---
200152024-01-24T19:14:16.730Z 12 240-259 179 179 179 314 314 314 F F F
200162024-01-24T19:14:16.730Z 13 260-279 186 185 186 331 326 331 F F F <---
200172024-01-24T19:14:16.730Z 14 280-299 186 185 186 331 326 331 F F F <---
200182024-01-24T19:14:16.731Z 15 300-319 184 184 184 324 324 324 F F F
200192024-01-24T19:14:16.731Z 16 320-339 184 184 184 324 324 324 F F F
200202024-01-24T19:14:16.731Z 17 340-359 185 185 185 326 326 326 F F F
200212024-01-24T19:14:16.731Z 18 360-379 180 180 180 317 317 317 F F F
200222024-01-24T19:14:16.731Z 19 380-399 186 185 186 331 326 331 F F F <---
200232024-01-24T19:14:16.731Z 20 400-419 186 178 186 331 311 331 T F T <---
200242024-01-24T19:14:16.731Z 21 420-439 186 186 186 327 327 327 F F F
200252024-01-24T19:14:16.731Z 22 440-459 186 186 186 327 327 327 T F T <---
200262024-01-24T19:14:16.731Z 23 460-479 183 183 183 322 322 322 F F F
200272024-01-24T19:14:16.731Z 24 480-499 174 174 174 302 302 302 F F F
200282024-01-24T19:14:16.731Z 25 500-519 177 177 177 310 310 310 F F F
200292024-01-24T19:14:16.731Z 26 520-539 181 181 181 318 318 318 F F F
200302024-01-24T19:14:16.731Z 27 540-559 184 184 184 325 325 325 T F T <---
200312024-01-24T19:14:16.731Z 28 560-579 183 183 183 322 322 322 F F F
200322024-01-24T19:14:16.732Z 29 580-599 182 182 182 321 321 321 F F F
200332024-01-24T19:14:16.732ZMax gen: 186, Max flush: 331
200342024-01-24T19:14:16.732ZError: Difference in extent metadata found!
200352024-01-24T19:14:16.736ZA Difference in extent metadata is expected here
200362024-01-24T19:14:16.736ZOn loop 93
200372024-01-24T19:14:16.736Z
200382024-01-24T19:14:16.736ZVerifying data now
200392024-01-24T19:14:21.880ZNo change after verify
200402024-01-24T19:14:21.880ZLoop: 93 Downstairs dump after verify (and repair):
200412024-01-24T19:14:21.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200422024-01-24T19:14:21.892Z 0 000-019 186 186 186 331 331 331 F F F
200432024-01-24T19:14:21.892Z 1 020-039 184 184 184 324 324 324 F F F
200442024-01-24T19:14:21.892Z 2 040-059 185 185 185 326 326 326 F F F
200452024-01-24T19:14:21.892Z 3 060-079 180 180 180 315 315 315 F F F
200462024-01-24T19:14:21.895Z 4 080-099 185 185 185 326 326 326 F F F
200472024-01-24T19:14:21.895Z 5 100-119 186 186 186 331 331 331 F F F
200482024-01-24T19:14:21.895Z 6 120-139 186 186 186 331 331 331 F F F
200492024-01-24T19:14:21.895Z 7 140-159 186 186 186 331 331 331 F F F
200502024-01-24T19:14:21.895Z 8 160-179 186 186 186 329 329 329 F F F
200512024-01-24T19:14:21.895Z 9 180-199 183 183 183 322 322 322 F F F
200522024-01-24T19:14:21.895Z 10 200-219 186 186 186 331 331 331 F F F
200532024-01-24T19:14:21.896Z 11 220-239 186 186 186 328 328 328 F F F
200542024-01-24T19:14:21.896Z 12 240-259 179 179 179 314 314 314 F F F
200552024-01-24T19:14:21.896Z 13 260-279 186 186 186 331 331 331 F F F
200562024-01-24T19:14:21.896Z 14 280-299 186 186 186 331 331 331 F F F
200572024-01-24T19:14:21.896Z 15 300-319 184 184 184 324 324 324 F F F
200582024-01-24T19:14:21.896Z 16 320-339 184 184 184 324 324 324 F F F
200592024-01-24T19:14:21.896Z 17 340-359 185 185 185 326 326 326 F F F
200602024-01-24T19:14:21.896Z 18 360-379 180 180 180 317 317 317 F F F
200612024-01-24T19:14:21.896Z 19 380-399 186 186 186 331 331 331 F F F
200622024-01-24T19:14:21.896Z 20 400-419 187 187 187 332 332 332 F F F
200632024-01-24T19:14:21.896Z 21 420-439 186 186 186 327 327 327 F F F
200642024-01-24T19:14:21.896Z 22 440-459 187 187 187 332 332 332 F F F
200652024-01-24T19:14:21.896Z 23 460-479 183 183 183 322 322 322 F F F
200662024-01-24T19:14:21.896Z 24 480-499 174 174 174 302 302 302 F F F
200672024-01-24T19:14:21.896Z 25 500-519 177 177 177 310 310 310 F F F
200682024-01-24T19:14:21.896Z 26 520-539 181 181 181 318 318 318 F F F
200692024-01-24T19:14:21.896Z 27 540-559 187 187 187 332 332 332 F F F
200702024-01-24T19:14:21.896Z 28 560-579 183 183 183 322 322 322 F F F
200712024-01-24T19:14:21.896Z 29 580-599 182 182 182 321 321 321 F F F
200722024-01-24T19:14:21.896ZMax gen: 187, Max flush: 332
200732024-01-24T19:14:21.903Z
200742024-01-24T19:14:21.903ZBegin loop 94
200752024-01-24T19:14:21.903ZDownstairs to restart: 2
200762024-01-24T19:14:21.929Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.608430704Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881}
200772024-01-24T19:14:21.934Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.608933822Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881}
200782024-01-24T19:14:21.934Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.60901428Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881}
200792024-01-24T19:14:21.934Z{"msg":"Crucible 9f8b9ed3-c9cb-43d4-a675-91a411f24428 has session id: bf0233d0-0052-41d6-80a4-839c2b8cf0cd","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.60905617Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200802024-01-24T19:14:21.934Z{"msg":"Upstairs opts: Upstairs UUID: 9f8b9ed3-c9cb-43d4-a675-91a411f24428, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609115799Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200812024-01-24T19:14:21.934Z{"msg":"Crucible stats registered with UUID: 9f8b9ed3-c9cb-43d4-a675-91a411f24428","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609158616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200822024-01-24T19:14:21.934ZCrucible runtime is spawned
200832024-01-24T19:14:21.934Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609544239Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881}
200842024-01-24T19:14:21.934Z{"msg":"9f8b9ed3-c9cb-43d4-a675-91a411f24428 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609710419Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200852024-01-24T19:14:21.934Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609771034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200862024-01-24T19:14:21.934Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609799366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200872024-01-24T19:14:21.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609837746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200882024-01-24T19:14:21.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609909762Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200892024-01-24T19:14:21.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609957417Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200902024-01-24T19:14:21.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.609998724Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200912024-01-24T19:14:21.935Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610048399Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200922024-01-24T19:14:21.935Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610091801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200932024-01-24T19:14:21.935Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610218616Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200942024-01-24T19:14:21.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610465736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200952024-01-24T19:14:21.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610514052Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200962024-01-24T19:14:21.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.610608761Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200972024-01-24T19:14:21.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.611718788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200982024-01-24T19:14:21.936Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.611783393Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
200992024-01-24T19:14:21.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.611953385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201002024-01-24T19:14:21.936Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.612038636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201012024-01-24T19:14:21.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.612260021Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201022024-01-24T19:14:21.942Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.612348866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201032024-01-24T19:14:21.943Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.612774949Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201042024-01-24T19:14:21.943Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.613189642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201052024-01-24T19:14:21.943Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.613739741Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201062024-01-24T19:14:21.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.613872337Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201072024-01-24T19:14:21.943Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.613928579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201082024-01-24T19:14:21.943Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.613971452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201092024-01-24T19:14:21.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614116374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201102024-01-24T19:14:21.943Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614144751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201112024-01-24T19:14:21.943Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614205332Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201122024-01-24T19:14:21.943Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.61473445Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201132024-01-24T19:14:21.944Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614810339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201142024-01-24T19:14:21.944Z{"msg":"[0]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 331, 331, 331, 329, 322, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.61485907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201152024-01-24T19:14:21.944Z{"msg":"[0]R generation[0..12]: [186, 184, 185, 180, 185, 186, 186, 186, 186, 183, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614918275Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201162024-01-24T19:14:21.944Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.614965527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201172024-01-24T19:14:21.944Z{"msg":"[1]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 331, 331, 331, 329, 322, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615017542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201182024-01-24T19:14:21.944Z{"msg":"[1]R generation[0..12]: [186, 184, 185, 180, 185, 186, 186, 186, 186, 183, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615044487Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201192024-01-24T19:14:21.944Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615091113Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201202024-01-24T19:14:21.944Z{"msg":"[2]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 331, 331, 331, 329, 322, 331, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615153433Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201212024-01-24T19:14:21.944Z{"msg":"[2]R generation[0..12]: [186, 184, 185, 180, 185, 186, 186, 186, 186, 183, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615180254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
201222024-01-24T19:14:21.944Zpid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201232024-01-24T19:14:21.944Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615216003Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201242024-01-24T19:14:21.944Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615239475Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201252024-01-24T19:14:21.944Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.61526892Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201262024-01-24T19:14:21.944Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615291619Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201272024-01-24T19:14:21.944Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615360354Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"":"downstairs","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201282024-01-24T19:14:21.944Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615387945Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201292024-01-24T19:14:21.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615429624Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"0","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201302024-01-24T19:14:21.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615455085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"1","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201312024-01-24T19:14:21.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615477331Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"client":"2","":"downstairs","session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201322024-01-24T19:14:21.945Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615520148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201332024-01-24T19:14:21.945Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615544044Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201342024-01-24T19:14:21.945Z{"msg":"9f8b9ed3-c9cb-43d4-a675-91a411f24428 is now active with session: bf0233d0-0052-41d6-80a4-839c2b8cf0cd","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615616368Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201352024-01-24T19:14:21.945Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615651922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881,"session_id":"bf0233d0-0052-41d6-80a4-839c2b8cf0cd"}
201362024-01-24T19:14:21.945Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:20.615712606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1881}
201372024-01-24T19:14:21.945ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201382024-01-24T19:14:21.945ZLoading write count information from file "/tmp/test_repair_verify.data"
201392024-01-24T19:14:21.945ZRun Repair workload
201402024-01-24T19:14:21.945Z01/30 Write block 272 len 36864 data: 20 21 17 17 14 16 16 19 19
201412024-01-24T19:14:21.948Z02/30 Flush
201422024-01-24T19:14:21.949Z03/30 Write block 214 len 20480 data: 9 9 11 9 8
201432024-01-24T19:14:21.949Z04/30 Read block 22 len 12288
201442024-01-24T19:14:21.949Z05/30 Read block 26 len 32768
201452024-01-24T19:14:21.949Z06/30 Read block 2 len 36864
201462024-01-24T19:14:21.951Z07/30 Read block 581 len 12288
201472024-01-24T19:14:21.954Z08/30 Flush
201482024-01-24T19:14:21.957Z09/30 Write block 399 len 28672 data: 13 14 15 14 9 9 11
201492024-01-24T19:14:21.957Z10/30 Read block 228 len 4096
201502024-01-24T19:14:21.960Z11/30 Write block 245 len 4096 data: 18
201512024-01-24T19:14:21.960Z12/30 Read block 151 len 36864
201522024-01-24T19:14:21.964Z13/30 Read block 81 len 40960
201532024-01-24T19:14:21.964Z14/30 Read block 525 len 8192
201542024-01-24T19:14:21.964Z15/30 Write block 571 len 32768 data: 16 16 15 15 16 17 12 14
201552024-01-24T19:14:21.967Z16/30 Flush
201562024-01-24T19:14:21.970Z17/30 Write block 319 len 32768 data: 16 15 14 14 14 14 16 11
201572024-01-24T19:14:21.970Z18/30 Read block 377 len 24576
201582024-01-24T19:14:21.973Z19/30 Read block 99 len 40960
201592024-01-24T19:14:21.977Z20/30 Write block 384 len 24576 data: 13 12 12 13 15 13
201602024-01-24T19:14:21.977Z21/30 Write block 284 len 8192 data: 15 16
201612024-01-24T19:14:21.980Z22/30 Read block 256 len 4096
201622024-01-24T19:14:21.980Z23/30 Write block 316 len 4096 data: 19
201632024-01-24T19:14:21.980Z24/30 Read block 342 len 24576
201642024-01-24T19:14:21.980Z25/30 Read block 291 len 12288
201652024-01-24T19:14:21.983Z26/30 Write block 245 len 24576 data: 19 19 19 17 18 14
201662024-01-24T19:14:21.983Z27/30 Write block 387 len 20480 data: 14 16 14 13 11
201672024-01-24T19:14:21.986Z28/30 Read block 253 len 16384
201682024-01-24T19:14:21.986Z29/30 Write block 112 len 8192 data: 15 13
201692024-01-24T19:14:21.989Z30/30 Write block 554 len 24576 data: 15 14 15 17 16 17
201702024-01-24T19:14:21.989Z----------------------------------------------------------------
201712024-01-24T19:14:21.990Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:28
201722024-01-24T19:14:21.990ZStates: Active Active Active
201732024-01-24T19:14:21.990ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
201742024-01-24T19:14:21.990Z 3 Acked 1002 Write 5 Done Done Sent false
201752024-01-24T19:14:21.990Z 4 Acked 1003 Read 3 Done Done Sent false
201762024-01-24T19:14:21.990Z 5 Acked 1004 Read 8 Done Done Sent false
201772024-01-24T19:14:21.990Z 6 Acked 1005 Read 9 Done Done Sent false
201782024-01-24T19:14:21.990Z 7 Acked 1006 Read 3 Done Done Sent false
201792024-01-24T19:14:21.990Z 8 Acked 1007 Flush 0 Done Done Sent false
201802024-01-24T19:14:21.990Z 9 Acked 1008 Write 7 Done Done Sent false
201812024-01-24T19:14:21.990Z 10 Acked 1009 Read 1 Done Done Sent false
201822024-01-24T19:14:21.990Z 11 Acked 1010 Write 1 Done Done Sent false
201832024-01-24T19:14:21.991Z 12 Acked 1011 Read 9 Done Done Sent false
201842024-01-24T19:14:21.991Z 13 Acked 1012 Read 10 Done Done Sent false
201852024-01-24T19:14:21.991Z 14 Acked 1013 Read 2 Done Done Sent false
201862024-01-24T19:14:21.991Z 15 Acked 1014 Write 8 Done Done Sent false
201872024-01-24T19:14:21.991Z 16 Acked 1015 Flush 0 Done Done Sent false
201882024-01-24T19:14:21.991Z 17 Acked 1016 Write 8 Done Done Sent false
201892024-01-24T19:14:21.991Z 18 Acked 1017 Read 6 Done Done Sent false
201902024-01-24T19:14:21.991Z 19 Acked 1018 Read 10 Done Done Sent false
201912024-01-24T19:14:21.991Z 20 Acked 1019 Write 6 Done Done Sent false
201922024-01-24T19:14:21.991Z 21 Acked 1020 Write 2 Done Done Sent false
201932024-01-24T19:14:21.991Z 22 Acked 1021 Read 1 Done Done Sent false
201942024-01-24T19:14:21.991Z 23 Acked 1022 Write 1 Done Done Sent false
201952024-01-24T19:14:21.991Z 24 Acked 1023 Read 6 Done Done Sent false
201962024-01-24T19:14:21.991Z 25 Acked 1024 Read 3 Done Done Sent false
201972024-01-24T19:14:21.991Z 26 Acked 1025 Write 6 Done Done Sent false
201982024-01-24T19:14:21.991Z 27 Acked 1026 Write 5 Done Done Sent false
201992024-01-24T19:14:21.991Z 28 Acked 1027 Read 4 Done Done Sent false
202002024-01-24T19:14:21.992Z 29 Acked 1028 Write 2 Done Done Sent false
202012024-01-24T19:14:21.992Z 30 Acked 1029 Write 6 Sent Sent Sent false
202022024-01-24T19:14:21.992Z STATES DS:0 DS:1 DS:2 TOTAL
202032024-01-24T19:14:21.992Z New 0 0 0 0
202042024-01-24T19:14:21.992Z Sent 1 1 28 30
202052024-01-24T19:14:21.992Z Done 27 27 0 54
202062024-01-24T19:14:21.992Z Skipped 0 0 0 0
202072024-01-24T19:14:21.992Z Error 0 0 0 0
202082024-01-24T19:14:21.992ZLast Flush: 1015 1015 1001
202092024-01-24T19:14:21.992ZDownstairs last five completed: 1001 1000
202102024-01-24T19:14:21.992ZUpstairs last five completed: 30 29 28 27 26
202112024-01-24T19:14:21.992ZWrote out file "/tmp/test_repair_verify.data"
202122024-01-24T19:14:21.996Z
202132024-01-24T19:14:22.002ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202142024-01-24T19:14:22.029ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202152024-01-24T19:14:22.029Z 0 000-019 186 186 186 331 331 331 F F F
202162024-01-24T19:14:22.030Z 1 020-039 184 184 184 324 324 324 F F F
202172024-01-24T19:14:22.030Z 2 040-059 185 185 185 326 326 326 F F F
202182024-01-24T19:14:22.030Z 3 060-079 180 180 180 315 315 315 F F F
202192024-01-24T19:14:22.033Z 4 080-099 185 185 185 326 326 326 F F F
202202024-01-24T19:14:22.033Z 5 100-119 186 186 186 331 331 331 T T F <---
202212024-01-24T19:14:22.033Z 6 120-139 186 186 186 331 331 331 F F F
202222024-01-24T19:14:22.033Z 7 140-159 186 186 186 331 331 331 F F F
202232024-01-24T19:14:22.033Z 8 160-179 186 186 186 329 329 329 F F F
202242024-01-24T19:14:22.033Z 9 180-199 183 183 183 322 322 322 F F F
202252024-01-24T19:14:22.035Z 10 200-219 188 188 186 334 334 331 F F F <---
202262024-01-24T19:14:22.035Z 11 220-239 186 186 186 328 328 328 F F F
202272024-01-24T19:14:22.035Z 12 240-259 188 188 179 335 335 314 T T F <---
202282024-01-24T19:14:22.035Z 13 260-279 188 188 188 333 333 333 F F F
202292024-01-24T19:14:22.035Z 14 280-299 188 188 188 333 333 333 T T F <---
202302024-01-24T19:14:22.035Z 15 300-319 184 184 184 324 324 324 T T F <---
202312024-01-24T19:14:22.035Z 16 320-339 184 184 184 324 324 324 T T F <---
202322024-01-24T19:14:22.035Z 17 340-359 185 185 185 326 326 326 F F F
202332024-01-24T19:14:22.035Z 18 360-379 180 180 180 317 317 317 F F F
202342024-01-24T19:14:22.035Z 19 380-399 188 188 186 335 335 331 T T F <---
202352024-01-24T19:14:22.035Z 20 400-419 188 188 187 335 335 332 F F F <---
202362024-01-24T19:14:22.035Z 21 420-439 186 186 186 327 327 327 F F F
202372024-01-24T19:14:22.035Z 22 440-459 187 187 187 332 332 332 F F F
202382024-01-24T19:14:22.035Z 23 460-479 183 183 183 322 322 322 F F F
202392024-01-24T19:14:22.036Z 24 480-499 174 174 174 302 302 302 F F F
202402024-01-24T19:14:22.036Z 25 500-519 177 177 177 310 310 310 F F F
202412024-01-24T19:14:22.036Z 26 520-539 181 181 181 318 318 318 F F F
202422024-01-24T19:14:22.036Z 27 540-559 187 187 187 332 332 332 T T F <---
202432024-01-24T19:14:22.036Z 28 560-579 188 188 183 335 335 322 F F F <---
202442024-01-24T19:14:22.036Z 29 580-599 182 182 182 321 321 321 F F F
202452024-01-24T19:14:22.036ZMax gen: 188, Max flush: 335
202462024-01-24T19:14:22.036ZError: Difference in extent metadata found!
202472024-01-24T19:14:22.039ZA Difference in extent metadata is expected here
202482024-01-24T19:14:22.039ZOn loop 94
202492024-01-24T19:14:22.039Z
202502024-01-24T19:14:22.041ZVerifying data now
202512024-01-24T19:14:36.077ZNo change after verify
202522024-01-24T19:14:36.077ZLoop: 94 Downstairs dump after verify (and repair):
202532024-01-24T19:14:36.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202542024-01-24T19:14:36.329Z 0 000-019 186 186 186 331 331 331 F F F
202552024-01-24T19:14:36.329Z 1 020-039 184 184 184 324 324 324 F F F
202562024-01-24T19:14:36.329Z 2 040-059 185 185 185 326 326 326 F F F
202572024-01-24T19:14:36.329Z 3 060-079 180 180 180 315 315 315 F F F
202582024-01-24T19:14:36.329Z 4 080-099 185 185 185 326 326 326 F F F
202592024-01-24T19:14:36.330Z 5 100-119 189 189 189 336 336 336 F F F
202602024-01-24T19:14:36.330Z 6 120-139 186 186 186 331 331 331 F F F
202612024-01-24T19:14:36.333Z 7 140-159 186 186 186 331 331 331 F F F
202622024-01-24T19:14:36.333Z 8 160-179 186 186 186 329 329 329 F F F
202632024-01-24T19:14:36.334Z 9 180-199 183 183 183 322 322 322 F F F
202642024-01-24T19:14:36.334Z 10 200-219 188 188 188 334 334 334 F F F
202652024-01-24T19:14:36.334Z 11 220-239 186 186 186 328 328 328 F F F
202662024-01-24T19:14:36.334Z 12 240-259 189 189 189 336 336 336 F F F
202672024-01-24T19:14:36.334Z 13 260-279 188 188 188 333 333 333 F F F
202682024-01-24T19:14:36.334Z 14 280-299 189 189 189 336 336 336 F F F
202692024-01-24T19:14:36.335Z 15 300-319 189 189 189 336 336 336 F F F
202702024-01-24T19:14:36.335Z 16 320-339 189 189 189 336 336 336 F F F
202712024-01-24T19:14:36.335Z 17 340-359 185 185 185 326 326 326 F F F
202722024-01-24T19:14:36.335Z 18 360-379 180 180 180 317 317 317 F F F
202732024-01-24T19:14:36.335Z 19 380-399 189 189 189 336 336 336 F F F
202742024-01-24T19:14:36.335Z 20 400-419 188 188 188 335 335 335 F F F
202752024-01-24T19:14:36.336Z 21 420-439 186 186 186 327 327 327 F F F
202762024-01-24T19:14:36.336Z 22 440-459 187 187 187 332 332 332 F F F
202772024-01-24T19:14:36.336Z 23 460-479 183 183 183 322 322 322 F F F
202782024-01-24T19:14:36.336Z 24 480-499 174 174 174 302 302 302 F F F
202792024-01-24T19:14:36.338Z 25 500-519 177 177 177 310 310 310 F F F
202802024-01-24T19:14:36.338Z 26 520-539 181 181 181 318 318 318 F F F
202812024-01-24T19:14:36.338Z 27 540-559 189 189 189 336 336 336 F F F
202822024-01-24T19:14:36.338Z 28 560-579 188 188 188 335 335 335 F F F
202832024-01-24T19:14:36.338Z 29 580-599 182 182 182 321 321 321 F F F
202842024-01-24T19:14:36.338ZMax gen: 189, Max flush: 336
202852024-01-24T19:14:36.339Z
202862024-01-24T19:14:36.339ZBegin loop 95
202872024-01-24T19:14:36.340ZDownstairs to restart: 0
202882024-01-24T19:14:36.340Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.737134501Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889}
202892024-01-24T19:14:36.340Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.737687956Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889}
202902024-01-24T19:14:36.340Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.737785801Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889}
202912024-01-24T19:14:36.340Z{"msg":"Crucible 6e2c1bd3-93e5-4ccc-8fbe-8ac7d023bdb7 has session id: dc7c7e3a-ca37-4a91-8f15-8bf2993ac633","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.737824664Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889Crucible runtime is spawned
202922024-01-24T19:14:36.340Z,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202932024-01-24T19:14:36.340Z{"msg":"Upstairs opts: Upstairs UUID: 6e2c1bd3-93e5-4ccc-8fbe-8ac7d023bdb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.737969151Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202942024-01-24T19:14:36.340Z{"msg":"Crucible stats registered with UUID: 6e2c1bd3-93e5-4ccc-8fbe-8ac7d023bdb7","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738067409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202952024-01-24T19:14:36.340Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738113646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889}
202962024-01-24T19:14:36.340Z{"msg":"6e2c1bd3-93e5-4ccc-8fbe-8ac7d023bdb7 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738242178Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202972024-01-24T19:14:36.340Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738312977Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202982024-01-24T19:14:36.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738359469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
202992024-01-24T19:14:36.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738426595Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203002024-01-24T19:14:36.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738472736Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203012024-01-24T19:14:36.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738718853Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203022024-01-24T19:14:36.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738837564Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203032024-01-24T19:14:36.342Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.738941359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203042024-01-24T19:14:36.342Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.739193539Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203052024-01-24T19:14:36.342Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.739275469Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203062024-01-24T19:14:36.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.739352976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203072024-01-24T19:14:36.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.739388272Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203082024-01-24T19:14:36.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.739416342Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203092024-01-24T19:14:36.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740666585Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203102024-01-24T19:14:36.352Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740736006Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203112024-01-24T19:14:36.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740763156Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203122024-01-24T19:14:36.353Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740826233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203132024-01-24T19:14:36.353Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740877489Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203142024-01-24T19:14:36.353Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.740960996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203152024-01-24T19:14:36.353Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.741676193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203162024-01-24T19:14:36.353Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.741739101Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203172024-01-24T19:14:36.353Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.742188838Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203182024-01-24T19:14:36.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.74271507Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203192024-01-24T19:14:36.353Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.742816421Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203202024-01-24T19:14:36.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.742977353Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203212024-01-24T19:14:36.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743018829Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203222024-01-24T19:14:36.353Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743098351Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203232024-01-24T19:14:36.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.74314666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203242024-01-24T19:14:36.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743278213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203252024-01-24T19:14:36.354Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.74332276Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203262024-01-24T19:14:36.354Z{"msg":"[0]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 336, 331, 331, 329, 322, 334, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743397051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203272024-01-24T19:14:36.354Z{"msg":"[0]R generation[0..12]: [186, 184, 185, 180, 185, 189, 186, 186, 186, 183, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743470237Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203282024-01-24T19:14:36.365Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
203292024-01-24T19:14:36.365Z,"time":"2024-01-24T19:14:25.743543839Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203302024-01-24T19:14:36.365Z{"msg":"[1]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 336, 331, 331, 329, 322, 334, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743593571Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203312024-01-24T19:14:36.366Z{"msg":"[1]R generation[0..12]: [186, 184, 185, 180, 185, 189, 186, 186, 186, 183, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743672478Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203322024-01-24T19:14:36.366Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743755628Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203332024-01-24T19:14:36.366Z{"msg":"[2]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 336, 331, 331, 329, 322, 334, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.74383746Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203342024-01-24T19:14:36.366Z{"msg":"[2]R generation[0..12]: [186, 184, 185, 180, 185, 189, 186, 186, 186, 183, 188, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.743921022Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203352024-01-24T19:14:36.366Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744001994Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203362024-01-24T19:14:36.366Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744050056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203372024-01-24T19:14:36.366Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203382024-01-24T19:14:36.367Z,"time":"2024-01-24T19:14:25.744126958Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203392024-01-24T19:14:36.367Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744204199Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203402024-01-24T19:14:36.367Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744233641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"":"downstairs","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203412024-01-24T19:14:36.367Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744256362Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203422024-01-24T19:14:36.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744298062Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"0","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203432024-01-24T19:14:36.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744323711Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"1","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203442024-01-24T19:14:36.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.74434556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"client":"2","":"downstairs","session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203452024-01-24T19:14:36.367Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744388913Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203462024-01-24T19:14:36.367Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744413258Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203472024-01-24T19:14:36.367Z{"msg":"6e2c1bd3-93e5-4ccc-8fbe-8ac7d023bdb7 is now active with session: dc7c7e3a-ca37-4a91-8f15-8bf2993ac633","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744435184Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203482024-01-24T19:14:36.367Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744475783Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889,"session_id":"dc7c7e3a-ca37-4a91-8f15-8bf2993ac633"}
203492024-01-24T19:14:36.368Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:25.744499636Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1889}
203502024-01-24T19:14:36.368ZLoading write count information from file "/tmp/test_repair_verify.data"
203512024-01-24T19:14:36.372ZRun Repair workload
203522024-01-24T19:14:36.372Z01/30 Write block 320 len 24576 data: 16 15 15 15 15 17
203532024-01-24T19:14:36.470Z02/30 Read block 385 len 40960
203542024-01-24T19:14:36.470Z03/30 Write block 306 len 8192 data: 9 11
203552024-01-24T19:14:36.470Z04/30 Read block 158 len 16384
203562024-01-24T19:14:36.470Z05/30 Write block 530 len 12288 data: 18 14 10
203572024-01-24T19:14:36.470Z06/30 Read block 120 len 36864
203582024-01-24T19:14:36.470Z07/30 Write block 572 len 28672 data: 17 16 16 17 18 13 15
203592024-01-24T19:14:36.470Z08/30 Write block 139 len 36864 data: 17 15 11 7 8 12 12 15 17
203602024-01-24T19:14:36.470Z09/30 Read block 415 len 40960
203612024-01-24T19:14:36.470Z10/30 Write block 445 len 36864 data: 8 7 8 9 9 8 10 11 13
203622024-01-24T19:14:36.470Z11/30 Write block 193 len 40960 data: 13 14 12 13 11 10 9 10 11 9
203632024-01-24T19:14:36.471Z12/30 Read block 235 len 36864
203642024-01-24T19:14:36.471Z13/30 Read block 272 len 20480
203652024-01-24T19:14:36.471Z14/30 Read block 172 len 32768
203662024-01-24T19:14:36.471Z15/30 Write block 413 len 24576 data: 12 10 9 9 12 14
203672024-01-24T19:14:36.471Z16/30 Read block 231 len 40960
203682024-01-24T19:14:36.471Z17/30 Write block 457 len 24576 data: 13 14 9 9 12 13
203692024-01-24T19:14:36.471Z18/30 Write block 272 len 8192 data: 21 22
203702024-01-24T19:14:36.471Z19/30 Read block 79 len 20480
203712024-01-24T19:14:36.471Z20/30 Write block 513 len 12288 data: 17 18 17
203722024-01-24T19:14:36.471Z21/30 Read block 190 len 16384
203732024-01-24T19:14:36.471Z22/30 Write block 511 len 16384 data: 17 17 18 19
203742024-01-24T19:14:36.471Z23/30 Read block 587 len 24576
203752024-01-24T19:14:36.471Z24/30 Read block 237 len 12288
203762024-01-24T19:14:36.471Z25/30 Write block 261 len 28672 data: 13 11 12 15 14 12 14
203772024-01-24T19:14:36.472Z26/30 Write block 105 len 36864 data: 19 19 22 22 23 22 20 16 14
203782024-01-24T19:14:36.472Z27/30 Write block 415 len 32768 data: 10 10 13 15 16 12 12 14
203792024-01-24T19:14:36.472Z28/30 Read block 445 len 32768
203802024-01-24T19:14:36.472Z29/30 Read block 91 len 12288
203812024-01-24T19:14:36.472Z30/30 Write block 522 len 20480 data: 13 13 17 18 18
203822024-01-24T19:14:36.472Z----------------------------------------------------------------
203832024-01-24T19:14:36.472Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
203842024-01-24T19:14:36.472ZStates: Active Active Active
203852024-01-24T19:14:36.472ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203862024-01-24T19:14:36.472Z 1 Acked 1000 Write 6 Done Done Done false
203872024-01-24T19:14:36.472Z 2 Acked 1001 Read 10 Done Done Done false
203882024-01-24T19:14:36.472Z 3 Acked 1002 Write 2 Done Done Done false
203892024-01-24T19:14:36.472Z 4 Acked 1003 Read 4 Done Done Done false
203902024-01-24T19:14:36.472Z 5 Acked 1004 Write 3 Done Done Done false
203912024-01-24T19:14:36.472Z 6 Acked 1005 Read 9 Done Done Done false
203922024-01-24T19:14:36.472Z 7 Acked 1006 Write 7 Done Done Done false
203932024-01-24T19:14:36.472Z 8 Acked 1007 Write 9 Done Done Done false
203942024-01-24T19:14:36.472Z 9 Acked 1008 Read 10 Done Done Done false
203952024-01-24T19:14:36.472Z 10 Acked 1009 Write 9 Sent Done Done false
203962024-01-24T19:14:36.472Z 11 Acked 1010 Write 10 Sent Done Done false
203972024-01-24T19:14:36.472Z 12 Acked 1011 Read 9 Sent Done Done false
203982024-01-24T19:14:36.472Z 13 Acked 1012 Read 5 Sent Done Done false
203992024-01-24T19:14:36.472Z 14 Acked 1013 Read 8 Sent Done Done false
204002024-01-24T19:14:36.473Z 15 Acked 1014 Write 6 Sent Done Done false
204012024-01-24T19:14:36.473Z 16 Acked 1015 Read 10 Sent Done Done false
204022024-01-24T19:14:36.473Z 17 Acked 1016 Write 6 Sent Done Done false
204032024-01-24T19:14:36.473Z 18 Acked 1017 Write 2 Sent Done Done false
204042024-01-24T19:14:36.473Z 19 Acked 1018 Read 5 Sent Done Done false
204052024-01-24T19:14:36.473Z 20 Acked 1019 Write 3 Sent Done Done false
204062024-01-24T19:14:36.473Z 21 Acked 1020 Read 4 Sent Done Done false
204072024-01-24T19:14:36.473Z 22 Acked 1021 Write 4 Sent Done Done false
204082024-01-24T19:14:36.473Z 23 Acked 1022 Read 6 Sent Done Done false
204092024-01-24T19:14:36.473Z 24 Acked 1023 Read 3 Sent Done Done false
204102024-01-24T19:14:36.473Z 25 Acked 1024 Write 7 Sent Done Done false
204112024-01-24T19:14:36.473Z 26 Acked 1025 Write 9 Sent Done Done false
204122024-01-24T19:14:36.473Z 27 Acked 1026 Write 8 Sent Done Done false
204132024-01-24T19:14:36.473Z 28 Acked 1027 Read 8 Sent Done Done false
204142024-01-24T19:14:36.473Z 29 Acked 1028 Read 3 Sent Done Done false
204152024-01-24T19:14:36.473Z 30 Acked 1029 Write 5 Sent Sent Sent false
204162024-01-24T19:14:36.473Z STATES DS:0 DS:1 DS:2 TOTAL
204172024-01-24T19:14:36.473Z New 0 0 0 0
204182024-01-24T19:14:36.473Z Sent 21 1 1 23
204192024-01-24T19:14:36.473Z Done 9 29 29 67
204202024-01-24T19:14:36.473Z Skipped 0 0 0 0
204212024-01-24T19:14:36.473Z Error 0 0 0 0
204222024-01-24T19:14:36.473ZLast Flush: 0 0 0
204232024-01-24T19:14:36.474ZDownstairs last five completed:
204242024-01-24T19:14:36.478ZUpstairs last five completed: 30 29 28 27 26
204252024-01-24T19:14:36.478ZWrote out file "/tmp/test_repair_verify.data"
204262024-01-24T19:14:36.478Z
204272024-01-24T19:14:36.478ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204282024-01-24T19:14:36.478ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204292024-01-24T19:14:36.478Z 0 000-019 186 186 186 331 331 331 F F F
204302024-01-24T19:14:36.478Z 1 020-039 184 184 184 324 324 324 F F F
204312024-01-24T19:14:36.478Z 2 040-059 185 185 185 326 326 326 F F F
204322024-01-24T19:14:36.478Z 3 060-079 180 180 180 315 315 315 F F F
204332024-01-24T19:14:36.478Z 4 080-099 185 185 185 326 326 326 F F F
204342024-01-24T19:14:36.479Z 5 100-119 189 189 189 336 336 336 F T T <---
204352024-01-24T19:14:36.479Z 6 120-139 186 186 186 331 331 331 T T T
204362024-01-24T19:14:36.479Z 7 140-159 186 186 186 331 331 331 T T T
204372024-01-24T19:14:36.479Z 8 160-179 186 186 186 329 329 329 F F F
204382024-01-24T19:14:36.479Z 9 180-199 183 183 183 322 322 322 F T T <---
204392024-01-24T19:14:36.479Z 10 200-219 188 188 188 334 334 334 F T T <---
204402024-01-24T19:14:36.479Z 11 220-239 186 186 186 328 328 328 F F F
204412024-01-24T19:14:36.479Z 12 240-259 189 189 189 336 336 336 F F F
204422024-01-24T19:14:36.479Z 13 260-279 188 188 188 333 333 333 F T T <---
204432024-01-24T19:14:36.479Z 14 280-299 189 189 189 336 336 336 F F F
204442024-01-24T19:14:36.479Z 15 300-319 189 189 189 336 336 336 T T T
204452024-01-24T19:14:36.479Z 16 320-339 189 189 189 336 336 336 T T T
204462024-01-24T19:14:36.479Z 17 340-359 185 185 185 326 326 326 F F F
204472024-01-24T19:14:36.479Z 18 360-379 180 180 180 317 317 317 F F F
204482024-01-24T19:14:36.480Z 19 380-399 189 189 189 336 336 336 F F F
204492024-01-24T19:14:36.480Z 20 400-419 188 188 188 335 335 335 F T T <---
204502024-01-24T19:14:36.480Z 21 420-439 186 186 186 327 327 327 F T T <---
204512024-01-24T19:14:36.480Z 22 440-459 187 187 187 332 332 332 F T T <---
204522024-01-24T19:14:36.480Z 23 460-479 183 183 183 322 322 322 F T T <---
204532024-01-24T19:14:36.513Z 24 480-499 174 174 174 302 302 302 F F F
204542024-01-24T19:14:36.514Z 25 500-519 177 177 177 310 310 310 F T T <---
204552024-01-24T19:14:36.514Z 26 520-539 181 181 181 318 318 318 T T T
204562024-01-24T19:14:36.515Z 27 540-559 189 189 189 336 336 336 F F F
204572024-01-24T19:14:36.515Z 28 560-579 188 188 188 335 335 335 T T T
204582024-01-24T19:14:36.515Z 29 580-599 182 182 182 321 321 321 F F F
204592024-01-24T19:14:36.515ZMax gen: 189, Max flush: 336
204602024-01-24T19:14:36.515ZError: Difference in extent metadata found!
204612024-01-24T19:14:36.515ZA Difference in extent metadata is expected here
204622024-01-24T19:14:36.515ZOn loop 95
204632024-01-24T19:14:36.515Z
204642024-01-24T19:14:36.515ZVerifying data now
204652024-01-24T19:14:36.515ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
204662024-01-24T19:14:36.516Z525,527c525,526
204672024-01-24T19:14:36.516Z< 12,
204682024-01-24T19:14:36.516Z< 12,
204692024-01-24T19:14:36.516Z< 16,
204702024-01-24T19:14:36.516Z---
204712024-01-24T19:14:36.516Z> 13,
204722024-01-24T19:14:36.517Z> 13,
204732024-01-24T19:14:36.517Z529c528,529
204742024-01-24T19:14:36.517Z< 17,
204752024-01-24T19:14:36.517Z---
204762024-01-24T19:14:36.517Z> 18,
204772024-01-24T19:14:36.517Z> 18,
204782024-01-24T19:14:36.517Zdiff found after verify
204792024-01-24T19:14:36.517ZLoop: 95 Downstairs dump after verify (and repair):
204802024-01-24T19:14:36.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204812024-01-24T19:14:36.517Z 0 000-019 186 186 186 331 331 331 F F F
204822024-01-24T19:14:36.517Z 1 020-039 184 184 184 324 324 324 F F F
204832024-01-24T19:14:36.517Z 2 040-059 185 185 185 326 326 326 F F F
204842024-01-24T19:14:36.519Z 3 060-079 180 180 180 315 315 315 F F F
204852024-01-24T19:14:36.519Z 4 080-099 185 185 185 326 326 326 F F F
204862024-01-24T19:14:36.519Z 5 100-119 190 190 190 337 337 337 F F F
204872024-01-24T19:14:36.519Z 6 120-139 190 190 190 337 337 337 F F F
204882024-01-24T19:14:36.519Z 7 140-159 190 190 190 337 337 337 F F F
204892024-01-24T19:14:36.519Z 8 160-179 186 186 186 329 329 329 F F F
204902024-01-24T19:14:36.519Z 9 180-199 190 190 190 337 337 337 F F F
204912024-01-24T19:14:36.519Z 10 200-219 190 190 190 337 337 337 F F F
204922024-01-24T19:14:36.519Z 11 220-239 186 186 186 328 328 328 F F F
204932024-01-24T19:14:36.519Z 12 240-259 189 189 189 336 336 336 F F F
204942024-01-24T19:14:36.520Z 13 260-279 190 190 190 337 337 337 F F F
204952024-01-24T19:14:36.520Z 14 280-299 189 189 189 336 336 336 F F F
204962024-01-24T19:14:36.520Z 15 300-319 190 190 190 337 337 337 F F F
204972024-01-24T19:14:36.520Z 16 320-339 190 190 190 337 337 337 F F F
204982024-01-24T19:14:36.520Z 17 340-359 185 185 185 326 326 326 F F F
204992024-01-24T19:14:36.520Z 18 360-379 180 180 180 317 317 317 F F F
205002024-01-24T19:14:36.520Z 19 380-399 189 189 189 336 336 336 F F F
205012024-01-24T19:14:36.520Z 20 400-419 190 190 190 337 337 337 F F F
205022024-01-24T19:14:36.520Z 21 420-439 190 190 190 337 337 337 F F F
205032024-01-24T19:14:36.520Z 22 440-459 190 190 190 337 337 337 F F F
205042024-01-24T19:14:36.521Z 23 460-479 190 190 190 337 337 337 F F F
205052024-01-24T19:14:36.521Z 24 480-499 174 174 174 302 302 302 F F F
205062024-01-24T19:14:36.521Z 25 500-519 190 190 190 337 337 337 F F F
205072024-01-24T19:14:36.521Z 26 520-539 190 190 190 337 337 337 F F F
205082024-01-24T19:14:36.521Z 27 540-559 189 189 189 336 336 336 F F F
205092024-01-24T19:14:36.521Z 28 560-579 190 190 190 337 337 337 F F F
205102024-01-24T19:14:36.521Z 29 580-599 182 182 182 321 321 321 F F F
205112024-01-24T19:14:36.521ZMax gen: 190, Max flush: 337
205122024-01-24T19:14:36.522Z
205132024-01-24T19:14:36.522ZBegin loop 96
205142024-01-24T19:14:36.522ZDownstairs to restart: 2
205152024-01-24T19:14:36.522Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.310542584Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900}
205162024-01-24T19:14:36.523Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311013001Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900}
205172024-01-24T19:14:36.523Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311042661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900}
205182024-01-24T19:14:36.523Z{"msg":"Crucible f831960d-986c-4ac6-b1fb-00b5881b2963 has session id: 8a81b25a-03db-450d-900f-5e9fbd1e81ab","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311095897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205192024-01-24T19:14:36.523Z{"msg":"Upstairs opts: Upstairs UUID: f831960d-986c-4ac6-b1fb-00b5881b2963, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311163903Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205202024-01-24T19:14:36.523Z{"msg":"Crucible stats registered with UUID: f831960d-986c-4ac6-b1fb-00b5881b2963","v":0Crucible runtime is spawned
205212024-01-24T19:14:36.523Z,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311393689Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205222024-01-24T19:14:36.523Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311470374Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900}
205232024-01-24T19:14:36.524Z{"msg":"f831960d-986c-4ac6-b1fb-00b5881b2963 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311659119Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205242024-01-24T19:14:36.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311737751Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205252024-01-24T19:14:36.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311787718Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205262024-01-24T19:14:36.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311887479Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205272024-01-24T19:14:36.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.311982796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205282024-01-24T19:14:36.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312123841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205292024-01-24T19:14:36.526Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312207363Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205302024-01-24T19:14:36.526Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312425623Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205312024-01-24T19:14:36.526Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312565289Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205322024-01-24T19:14:36.526Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312625042Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205332024-01-24T19:14:36.527Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312704527Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205342024-01-24T19:14:36.527Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.312753015Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205352024-01-24T19:14:36.527Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.31278757Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205362024-01-24T19:14:36.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.313859546Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205372024-01-24T19:14:36.527Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.313920707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205382024-01-24T19:14:36.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.314053898Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205392024-01-24T19:14:36.527Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.314100254Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205402024-01-24T19:14:36.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.314142409Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205412024-01-24T19:14:36.527Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.314220247Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205422024-01-24T19:14:36.528Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.314918678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205432024-01-24T19:14:36.528Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.315257376Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205442024-01-24T19:14:36.528Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.315411897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205452024-01-24T19:14:36.528Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.315732788Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205462024-01-24T19:14:36.528Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.31582224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205472024-01-24T19:14:36.528Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.315900193Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205482024-01-24T19:14:36.528Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316243509Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205492024-01-24T19:14:36.528Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316295213Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205502024-01-24T19:14:36.528Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316360452Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205512024-01-24T19:14:36.528Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316406036Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205522024-01-24T19:14:36.528Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316452502Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205532024-01-24T19:14:36.627Z{"msg":"[0]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 337, 337, 337, 329, 337, 337, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.3164996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205542024-01-24T19:14:36.627Z{"msg":"[0]R generation[0..12]: [186, 184, 185, 180, 185, 190, 190, 190, 186, 190, 190, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316547718Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205552024-01-24T19:14:36.627Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.31659573Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205562024-01-24T19:14:36.627Z{"msg":"[1]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 337, 337, 337, 329, 337, 337, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316643207Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205572024-01-24T19:14:36.627Z{"msg":"[1]R generation[0..12]: [186, 184, 185, 180, 185, 190, 190, 190, 186, 190, 190, 186]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
205582024-01-24T19:14:36.627Z2024-01-24T19:14:32.316690905Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205592024-01-24T19:14:36.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316808548Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205602024-01-24T19:14:36.627Z{"msg":"[2]R flush_numbers[0..12]: [331, 324, 326, 315, 326, 337, 337, 337, 329, 337, 337, 328]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316862457Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205612024-01-24T19:14:36.629Z{"msg":"[2]R generation[0..12]: [186, 184, 185, 180, 185, 190, 190, 190, 186, 190, 190, 186]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316911734Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205622024-01-24T19:14:36.629Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.316960141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205632024-01-24T19:14:36.629Z{"msg":"Max found gen is 191","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317010186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205642024-01-24T19:14:36.629Z{"msg":"Generation requested: 192 >= found:191","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317036257Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205652024-01-24T19:14:36.629Z{"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317057883Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205662024-01-24T19:14:36.629ZError: Difference in extent metadata found!
205672024-01-24T19:14:36.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317102796Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205682024-01-24T19:14:36.630Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317128323Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205692024-01-24T19:14:36.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317149161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205702024-01-24T19:14:36.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317170663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205712024-01-24T19:14:36.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317192719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205722024-01-24T19:14:36.631Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317214587Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205732024-01-24T19:14:36.631Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317234963Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205742024-01-24T19:14:36.631Z{"msg":"f831960d-986c-4ac6-b1fb-00b5881b2963 is now active with session: 8a81b25a-03db-450d-900f-5e9fbd1e81ab","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317256094Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205752024-01-24T19:14:36.631Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.31727599Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900,"session_id":"8a81b25a-03db-450d-900f-5e9fbd1e81ab"}
205762024-01-24T19:14:36.631Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:32.317296634Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1900}
205772024-01-24T19:14:36.631ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205782024-01-24T19:14:36.631ZLoading write count information from file "/tmp/test_repair_verify.data"
205792024-01-24T19:14:36.631ZRun Repair workload
205802024-01-24T19:14:36.631Z01/30 Write block 165 len 8192 data: 23 20
205812024-01-24T19:14:36.631Z02/30 Write block 461 len 16384 data: 13 14 11 10
205822024-01-24T19:14:36.631Z03/30 Write block 70 len 32768 data: 8 9 10 7 9 10 11 13
205832024-01-24T19:14:36.632Z04/30 Read block 397 len 8192
205842024-01-24T19:14:36.632Z05/30 Write block 80 len 32768 data: 15 16 13 15 16 13 14 11
205852024-01-24T19:14:36.632Z06/30 Read block 317 len 20480
205862024-01-24T19:14:36.632Z07/30 Read block 509 len 4096
205872024-01-24T19:14:36.632Z08/30 Read block 102 len 4096
205882024-01-24T19:14:36.632Z09/30 Write block 405 len 28672 data: 12 15 14 16 10 12 14
205892024-01-24T19:14:36.634Z10/30 Read block 21 len 28672
205902024-01-24T19:14:36.634Z11/30 Write block 206 len 32768 data: 7 8 9 11 15 13 13 12
205912024-01-24T19:14:36.634Z12/30 Read block 179 len 8192
205922024-01-24T19:14:36.634Z13/30 Write block 416 len 8192 data: 11 14
205932024-01-24T19:14:36.634Z14/30 Read block 285 len 40960
205942024-01-24T19:14:36.634Z15/30 Write block 148 len 4096 data: 17
205952024-01-24T19:14:36.634Z16/30 Write block 220 len 28672 data: 6 10 12 8 10 12 13
205962024-01-24T19:14:36.634Z17/30 Read block 543 len 40960
205972024-01-24T19:14:36.634Z18/30 Read block 563 len 32768
205982024-01-24T19:14:36.634Z19/30 Read block 322 len 4096
205992024-01-24T19:14:36.635Z20/30 Flush
206002024-01-24T19:14:36.635Z21/30 Write block 19 len 4096 data: 11
206012024-01-24T19:14:36.635Z22/30 Read block 270 len 24576
206022024-01-24T19:14:36.635Z23/30 Write block 374 len 32768 data: 16 19 17 17 12 12 15 17
206032024-01-24T19:14:36.635Z24/30 Read block 11 len 40960
206042024-01-24T19:14:36.635Z25/30 Write block 48 len 4096 data: 16
206052024-01-24T19:14:36.635Z26/30 Read block 278 len 40960
206062024-01-24T19:14:36.635Z27/30 Write block 201 len 36864 data: 12 10 10 12 11 8 9 10 12
206072024-01-24T19:14:36.635Z28/30 Read block 461 len 4096
206082024-01-24T19:14:36.635Z29/30 Write block 100 len 28672 data: 10 8 10 13 14 20 20
206092024-01-24T19:14:36.635Z30/30 Read block 573 len 32768
206102024-01-24T19:14:36.635Z----------------------------------------------------------------
206112024-01-24T19:14:36.635Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
206122024-01-24T19:14:36.636ZStates: Active Active Active
206132024-01-24T19:14:36.636ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206142024-01-24T19:14:36.636Z 1 Acked 1000 Write 2 Done Done Done false
206152024-01-24T19:14:36.637Z 2 Acked 1001 Write 4 Done Done Done false
206162024-01-24T19:14:36.637Z 3 Acked 1002 Write 8 Done Done Done false
206172024-01-24T19:14:36.638Z 4 Acked 1003 Read 2 Done Done Done false
206182024-01-24T19:14:36.638Z 5 Acked 1004 Write 8 Done Done Done false
206192024-01-24T19:14:36.638Z 6 Acked 1005 Read 5 Done Done Done false
206202024-01-24T19:14:36.638Z 7 Acked 1006 Read 1 Done Done Done false
206212024-01-24T19:14:36.638Z 8 Acked 1007 Read 1 Done Done Done false
206222024-01-24T19:14:36.638Z 9 Acked 1008 Write 7 Done Done Sent false
206232024-01-24T19:14:36.638Z 10 Acked 1009 Read 7 Done Done Sent false
206242024-01-24T19:14:36.638Z 11 Acked 1010 Write 8 Done Done Sent false
206252024-01-24T19:14:36.638Z 12 Acked 1011 Read 2 Done Done Sent false
206262024-01-24T19:14:36.638Z 13 Acked 1012 Write 2 Done Done Sent false
206272024-01-24T19:14:36.638Z 14 Acked 1013 Read 10 Done Done Sent false
206282024-01-24T19:14:36.638Z 15 Acked 1014 Write 1 Done Done Sent false
206292024-01-24T19:14:36.638Z 16 Acked 1015 Write 7 Done Done Sent false
206302024-01-24T19:14:36.638Z 17 Acked 1016 Read 10 Done Done Sent false
206312024-01-24T19:14:36.638Z 18 Acked 1017 Read 8 Done Done Sent false
206322024-01-24T19:14:36.638Z 19 Acked 1018 Read 1 Done Done Sent false
206332024-01-24T19:14:36.638Z 20 Acked 1019 Flush 0 Done Done Sent false
206342024-01-24T19:14:36.638Z 21 Acked 1020 Write 1 Done Done Sent false
206352024-01-24T19:14:36.638Z 22 Acked 1021 Read 6 Done Done Sent false
206362024-01-24T19:14:36.638Z 23 Acked 1022 Write 8 Done Done Sent false
206372024-01-24T19:14:36.639Z 24 Acked 1023 Read 10 Done Done Sent false
206382024-01-24T19:14:36.639Z 25 Acked 1024 Write 1 Done Done Sent false
206392024-01-24T19:14:36.639Z 26 Acked 1025 Read 10 Done Done Sent false
206402024-01-24T19:14:36.639Z 27 Acked 1026 Write 9 Done Done Sent false
206412024-01-24T19:14:36.639Z 28 Acked 1027 Read 1 Done Done Sent false
206422024-01-24T19:14:36.639Z 29 Acked 1028 Write 7 Done Done Sent false
206432024-01-24T19:14:36.639Z 30 Acked 1029 Read 8 Done Sent Sent false
206442024-01-24T19:14:36.639Z STATES DS:0 DS:1 DS:2 TOTAL
206452024-01-24T19:14:36.639Z New 0 0 0 0
206462024-01-24T19:14:36.639Z Sent 0 1 22 23
206472024-01-24T19:14:36.639Z Done 30 29 8 67
206482024-01-24T19:14:36.639Z Skipped 0 0 0 0
206492024-01-24T19:14:36.639Z Error 0 0 0 0
206502024-01-24T19:14:36.639ZLast Flush: 1019 1019 0
206512024-01-24T19:14:36.639ZDownstairs last five completed:
206522024-01-24T19:14:36.639ZUpstairs last five completed: 30 29 28 27 26
206532024-01-24T19:14:36.671ZWrote out file "/tmp/test_repair_verify.data"
206542024-01-24T19:14:36.671Z
206552024-01-24T19:14:36.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206562024-01-24T19:14:36.672ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206572024-01-24T19:14:36.672Z 0 000-019 186 186 186 331 331 331 T T F <---
206582024-01-24T19:14:36.672Z 1 020-039 184 184 184 324 324 324 F F F
206592024-01-24T19:14:36.672Z 2 040-059 185 185 185 326 326 326 T T F <---
206602024-01-24T19:14:36.673Z 3 060-079 192 192 180 338 338 315 F F T <---
206612024-01-24T19:14:36.673Z 4 080-099 192 192 185 338 338 326 F F T <---
206622024-01-24T19:14:36.673Z 5 100-119 190 190 190 337 337 337 T T F <---
206632024-01-24T19:14:36.673Z 6 120-139 190 190 190 337 337 337 F F F
206642024-01-24T19:14:36.673Z 7 140-159 192 192 190 338 338 337 F F F <---
206652024-01-24T19:14:36.673Z 8 160-179 192 192 186 338 338 329 F F T <---
206662024-01-24T19:14:36.673Z 9 180-199 190 190 190 337 337 337 F F F
206672024-01-24T19:14:36.673Z 10 200-219 192 192 190 338 338 337 T T F <---
206682024-01-24T19:14:36.673Z 11 220-239 192 192 186 338 338 328 F F F <---
206692024-01-24T19:14:36.674Z 12 240-259 189 189 189 336 336 336 F F F
206702024-01-24T19:14:36.674Z 13 260-279 190 190 190 337 337 337 F F F
206712024-01-24T19:14:36.674Z 14 280-299 189 189 189 336 336 336 F F F
206722024-01-24T19:14:36.674Z 15 300-319 190 190 190 337 337 337 F F F
206732024-01-24T19:14:36.674Z 16 320-339 190 190 190 337 337 337 F F F
206742024-01-24T19:14:36.674Z 17 340-359 185 185 185 326 326 326 F F F
206752024-01-24T19:14:36.674Z 18 360-379 180 180 180 317 317 317 T T F <---
206762024-01-24T19:14:36.674Z 19 380-399 189 189 189 336 336 336 T T F <---
206772024-01-24T19:14:36.674Z 20 400-419 192 192 190 338 338 337 F F F <---
206782024-01-24T19:14:36.674Z 21 420-439 190 190 190 337 337 337 F F F
206792024-01-24T19:14:36.674Z 22 440-459 190 190 190 337 337 337 F F F
206802024-01-24T19:14:36.674Z 23 460-479 192 192 190 338 338 337 F F T <---
206812024-01-24T19:14:36.674Z 24 480-499 174 174 174 302 302 302 F F F
206822024-01-24T19:14:36.674Z 25 500-519 190 190 190 337 337 337 F F F
206832024-01-24T19:14:36.674Z 26 520-539 190 190 190 337 337 337 F F F
206842024-01-24T19:14:36.675Z 27 540-559 189 189 189 336 336 336 F F F
206852024-01-24T19:14:36.675Z 28 560-579 190 190 190 337 337 337 F F F
206862024-01-24T19:14:36.675Z 29 580-599 182 182 182 321 321 321 F F F
206872024-01-24T19:14:36.675ZMax gen: 192, Max flush: 338
206882024-01-24T19:14:36.675ZA Difference in extent metadata is expected here
206892024-01-24T19:14:36.675ZOn loop 96
206902024-01-24T19:14:36.675Z
206912024-01-24T19:14:36.675ZVerifying data now
206922024-01-24T19:14:39.914ZNo change after verify
206932024-01-24T19:14:39.914ZLoop: 96 Downstairs dump after verify (and repair):
206942024-01-24T19:14:39.951ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206952024-01-24T19:14:39.951Z 0 000-019 193 193 193 339 339 339 F F F
206962024-01-24T19:14:39.952Z 1 020-039 184 184 184 324 324 324 F F F
206972024-01-24T19:14:39.952Z 2 040-059 193 193 193 339 339 339 F F F
206982024-01-24T19:14:39.958Z 3 060-079 192 192 192 338 338 338 F F F
206992024-01-24T19:14:39.958Z 4 080-099 192 192 192 338 338 338 F F F
207002024-01-24T19:14:39.958Z 5 100-119 193 193 193 339 339 339 F F F
207012024-01-24T19:14:39.958Z 6 120-139 190 190 190 337 337 337 F F F
207022024-01-24T19:14:39.958Z 7 140-159 192 192 192 338 338 338 F F F
207032024-01-24T19:14:39.959Z 8 160-179 192 192 192 338 338 338 F F F
207042024-01-24T19:14:39.959Z 9 180-199 190 190 190 337 337 337 F F F
207052024-01-24T19:14:39.959Z 10 200-219 193 193 193 339 339 339 F F F
207062024-01-24T19:14:39.959Z 11 220-239 192 192 192 338 338 338 F F F
207072024-01-24T19:14:39.959Z 12 240-259 189 189 189 336 336 336 F F F
207082024-01-24T19:14:39.959Z 13 260-279 190 190 190 337 337 337 F F F
207092024-01-24T19:14:39.959Z 14 280-299 189 189 189 336 336 336 F F F
207102024-01-24T19:14:39.959Z 15 300-319 190 190 190 337 337 337 F F F
207112024-01-24T19:14:39.959Z 16 320-339 190 190 190 337 337 337 F F F
207122024-01-24T19:14:39.959Z 17 340-359 185 185 185 326 326 326 F F F
207132024-01-24T19:14:39.959Z 18 360-379 193 193 193 339 339 339 F F F
207142024-01-24T19:14:39.959Z 19 380-399 193 193 193 339 339 339 F F F
207152024-01-24T19:14:39.959Z 20 400-419 192 192 192 338 338 338 F F F
207162024-01-24T19:14:39.960Z 21 420-439 190 190 190 337 337 337 F F F
207172024-01-24T19:14:39.960Z 22 440-459 190 190 190 337 337 337 F F F
207182024-01-24T19:14:39.960Z 23 460-479 192 192 192 338 338 338 F F F
207192024-01-24T19:14:39.960Z 24 480-499 174 174 174 302 302 302 F F F
207202024-01-24T19:14:39.960Z 25 500-519 190 190 190 337 337 337 F F F
207212024-01-24T19:14:39.960Z 26 520-539 190 190 190 337 337 337 F F F
207222024-01-24T19:14:39.963Z 27 540-559 189 189 189 336 336 336 F F F
207232024-01-24T19:14:39.963Z 28 560-579 190 190 190 337 337 337 F F F
207242024-01-24T19:14:39.963Z 29 580-599 182 182 182 321 321 321 F F F
207252024-01-24T19:14:39.963ZMax gen: 193, Max flush: 339
207262024-01-24T19:14:39.963Z
207272024-01-24T19:14:39.963ZBegin loop 97
207282024-01-24T19:14:39.963ZDownstairs to restart: 1
207292024-01-24T19:14:39.963Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62401922Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908}
207302024-01-24T19:14:39.963Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.624545076Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908}
207312024-01-24T19:14:39.963Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.624622459Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908}
207322024-01-24T19:14:39.963Z{"msg":"Crucible 0a31358d-d447-44ff-a739-717dcbaf2b28 has session id: 551d8b36-89da-4317-aa8c-355776917cd9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.624650192Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207332024-01-24T19:14:39.964Z{"msg":"Upstairs opts: Upstairs UUID: 0a31358d-d447-44ff-a739-717dcbaf2b28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.624735272Z","hostname":"Crucible runtime is spawned
207342024-01-24T19:14:39.964Zip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207352024-01-24T19:14:39.964Z{"msg":"Crucible stats registered with UUID: 0a31358d-d447-44ff-a739-717dcbaf2b28","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62492141Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207362024-01-24T19:14:39.964Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625027486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908}
207372024-01-24T19:14:39.968Z{"msg":"0a31358d-d447-44ff-a739-717dcbaf2b28 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625213973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207382024-01-24T19:14:39.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625275035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207392024-01-24T19:14:39.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625303966Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207402024-01-24T19:14:39.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625332648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207412024-01-24T19:14:39.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625375085Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207422024-01-24T19:14:39.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625399659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207432024-01-24T19:14:39.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625474008Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207442024-01-24T19:14:39.969Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625510643Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207452024-01-24T19:14:39.969Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62559738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207462024-01-24T19:14:39.969Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.625643663Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207472024-01-24T19:14:39.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62595037Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207482024-01-24T19:14:39.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.626199738Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207492024-01-24T19:14:39.970Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.626315866Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207502024-01-24T19:14:40.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.627558888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207512024-01-24T19:14:40.040Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.627718778Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207522024-01-24T19:14:40.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62778878Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207532024-01-24T19:14:40.040Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.62783373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207542024-01-24T19:14:40.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.628641606Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207552024-01-24T19:14:40.040Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.628728996Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207562024-01-24T19:14:40.042Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.628829505Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207572024-01-24T19:14:40.042Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.628960224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207582024-01-24T19:14:40.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629719214Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207592024-01-24T19:14:40.042Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629806579Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207602024-01-24T19:14:40.042Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629844971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207612024-01-24T19:14:40.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629891403Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207622024-01-24T19:14:40.043Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629917731Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207632024-01-24T19:14:40.043Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.629994265Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207642024-01-24T19:14:40.043Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.630270359Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207652024-01-24T19:14:40.043Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.630998295Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207662024-01-24T19:14:40.043Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631063971Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207672024-01-24T19:14:40.044Z{"msg":"[0]R flush_numbers[0..12]: [339, 324, 339, 338, 338, 339, 337, 338, 338, 337, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631093607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207682024-01-24T19:14:40.045Z{"msg":"[0]R generation[0..12]: [193, 184, 193, 192, 192, 193, 190, 192, 192, 190, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631117982Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207692024-01-24T19:14:40.047Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631163692Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207702024-01-24T19:14:40.047Z{"msg":"[1]R flush_numbers[0..12]: [339, 324, 339, 338, 338, 339, 337, 338, 338, 337, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631191398Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207712024-01-24T19:14:40.047Z{"msg":"[1]R generation[0..12]: [193, 184, 193, 192, 192, 193, 190, 192, 192, 190, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631236071Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207722024-01-24T19:14:40.047Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631263057Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207732024-01-24T19:14:40.047Z{"msg":"[2]R flush_numbers[0..12]: [339, 324, 339, 338, 338, 339, 337, 338, 338, 337, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631307148Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207742024-01-24T19:14:40.048Z{"msg":"[2]R generation[0..12]: [193, 184, 193, 192, 192, 193, 190, 192, 192, 190, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631333678Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207752024-01-24T19:14:40.048Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
207762024-01-24T19:14:40.048Z,"time":"2024-01-24T19:14:38.631377666Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207772024-01-24T19:14:40.049Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631404222Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207782024-01-24T19:14:40.049Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631427041Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207792024-01-24T19:14:40.049Z{"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631468875Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207802024-01-24T19:14:40.049Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631494309Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207812024-01-24T19:14:40.049Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631538298Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207822024-01-24T19:14:40.049Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631561726Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207832024-01-24T19:14:40.049Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631584161Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207842024-01-24T19:14:40.050Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631626641Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207852024-01-24T19:14:40.050Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631652176Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207862024-01-24T19:14:40.050Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631673226Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207872024-01-24T19:14:40.054Z{"msg":"0a31358d-d447-44ff-a739-717dcbaf2b28 is now active with session: 551d8b36-89da-4317-aa8c-355776917cd9","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631716328Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207882024-01-24T19:14:40.054Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631739563Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908,"session_id":"551d8b36-89da-4317-aa8c-355776917cd9"}
207892024-01-24T19:14:40.055Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:38.631761707Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1908}
207902024-01-24T19:14:40.055ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207912024-01-24T19:14:40.055ZLoading write count information from file "/tmp/test_repair_verify.data"
207922024-01-24T19:14:40.055ZRun Repair workload
207932024-01-24T19:14:40.055Z01/30 Write block 401 len 8192 data: 16 15
207942024-01-24T19:14:40.055Z02/30 Flush
207952024-01-24T19:14:40.055Z03/30 Write block 188 len 36864 data: 12 12 12 12 13 14 15 13 14
207962024-01-24T19:14:40.055Z04/30 Write block 421 len 16384 data: 13 15 14 14
207972024-01-24T19:14:40.055Z05/30 Read block 260 len 28672
207982024-01-24T19:14:40.055Z06/30 Write block 465 len 36864 data: 11 11 13 13 10 12 15 19 14
207992024-01-24T19:14:40.055Z07/30 Read block 20 len 32768
208002024-01-24T19:14:40.055Z08/30 Read block 497 len 16384
208012024-01-24T19:14:40.055Z09/30 Write block 492 len 8192 data: 11 12
208022024-01-24T19:14:40.055Z10/30 Write block 120 len 20480 data: 13 14 15 13 16
208032024-01-24T19:14:40.055Z11/30 Read block 326 len 32768
208042024-01-24T19:14:40.055Z12/30 Read block 97 len 24576
208052024-01-24T19:14:40.055Z13/30 Read block 107 len 12288
208062024-01-24T19:14:40.055Z14/30 Write block 2 len 32768 data: 8 10 10 10 12 13 15 12
208072024-01-24T19:14:40.058Z15/30 Read block 411 len 12288
208082024-01-24T19:14:40.058Z16/30 Read block 527 len 32768
208092024-01-24T19:14:40.058Z17/30 Read block 200 len 16384
208102024-01-24T19:14:40.059Z18/30 Read block 217 len 36864
208112024-01-24T19:14:40.059Z19/30 Write block 32 len 32768 data: 17 17 17 10 10 7 7 9
208122024-01-24T19:14:40.059Z20/30 Read block 449 len 8192
208132024-01-24T19:14:40.079Z21/30 Write block 96 len 8192 data: 11 13
208142024-01-24T19:14:40.080Z22/30 Read block 108 len 40960
208152024-01-24T19:14:40.080Z23/30 Read block 124 len 24576
208162024-01-24T19:14:40.080Z24/30 Write block 52 len 8192 data: 16 15
208172024-01-24T19:14:40.080Z25/30 Read block 534 len 36864
208182024-01-24T19:14:40.080Z26/30 Read block 576 len 32768
208192024-01-24T19:14:40.080Z27/30 Read block 524 len 36864
208202024-01-24T19:14:40.081Z28/30 Write block 98 len 40960 data: 12 10 11 9 11 14 15 21 21 23
208212024-01-24T19:14:40.081Z29/30 Read block 529 len 28672
208222024-01-24T19:14:40.081Z30/30 Read block 394 len 4096
208232024-01-24T19:14:40.081Z----------------------------------------------------------------
208242024-01-24T19:14:40.081Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
208252024-01-24T19:14:40.081ZStates: Active Active Active
208262024-01-24T19:14:40.081ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208272024-01-24T19:14:40.081Z 1 Acked 1000 Write 2 Done Sent Done false
208282024-01-24T19:14:40.082Z 2 Acked 1001 Flush 0 Done Sent Done false
208292024-01-24T19:14:40.082Z 3 Acked 1002 Write 9 Done Sent Done false
208302024-01-24T19:14:40.082Z 4 Acked 1003 Write 4 Done Sent Done false
208312024-01-24T19:14:40.082Z 5 Acked 1004 Read 7 Done Sent Done false
208322024-01-24T19:14:40.083Z 6 Acked 1005 Write 9 Done Sent Done false
208332024-01-24T19:14:40.083Z 7 Acked 1006 Read 8 Done Sent Done false
208342024-01-24T19:14:40.083Z 8 Acked 1007 Read 4 Done Sent Done false
208352024-01-24T19:14:40.083Z 9 Acked 1008 Write 2 Done Sent Done false
208362024-01-24T19:14:40.084Z 10 Acked 1009 Write 5 Done Sent Done false
208372024-01-24T19:14:40.084Z 11 Acked 1010 Read 8 Done Sent Done false
208382024-01-24T19:14:40.085Z 12 Acked 1011 Read 6 Done Sent Done false
208392024-01-24T19:14:40.085Z 13 Acked 1012 Read 3 Done Sent Done false
208402024-01-24T19:14:40.085Z 14 Acked 1013 Write 8 Done Sent Done false
208412024-01-24T19:14:40.085Z 15 Acked 1014 Read 3 Done Sent Done false
208422024-01-24T19:14:40.085Z 16 Acked 1015 Read 8 Done Sent Done false
208432024-01-24T19:14:40.085Z 17 Acked 1016 Read 4 Done Sent Done false
208442024-01-24T19:14:40.085Z 18 Acked 1017 Read 9 Done Sent Done false
208452024-01-24T19:14:40.085Z 19 Acked 1018 Write 8 Done Sent Done false
208462024-01-24T19:14:40.085Z 20 Acked 1019 Read 2 Done Sent Done false
208472024-01-24T19:14:40.086Z 21 Acked 1020 Write 2 Done Sent Done false
208482024-01-24T19:14:40.086Z 22 Acked 1021 Read 10 Done Sent Done false
208492024-01-24T19:14:40.086Z 23 Acked 1022 Read 6 Done Sent Done false
208502024-01-24T19:14:40.086Z 24 Acked 1023 Write 2 Done Sent Done false
208512024-01-24T19:14:40.086Z 25 Acked 1024 Read 9 Done Sent Done false
208522024-01-24T19:14:40.086Z 26 Acked 1025 Read 8 Done Sent Done false
208532024-01-24T19:14:40.086Z 27 Acked 1026 Read 9 Done Sent Done false
208542024-01-24T19:14:40.087Z 28 Acked 1027 Write 10 Done Sent Done false
208552024-01-24T19:14:40.088Z 29 Acked 1028 Read 7 Done Sent Done false
208562024-01-24T19:14:40.088Z 30 Acked 1029 Read 1 Done Sent Sent false
208572024-01-24T19:14:40.089Z STATES DS:0 DS:1 DS:2 TOTAL
208582024-01-24T19:14:40.089Z New 0 0 0 0
208592024-01-24T19:14:40.089Z Sent 0 30 1 31
208602024-01-24T19:14:40.089Z Done 30 0 29 59
208612024-01-24T19:14:40.089Z Skipped 0 0 0 0
208622024-01-24T19:14:40.089Z Error 0 0 0 0
208632024-01-24T19:14:40.090ZLast Flush: 1001 0 1001
208642024-01-24T19:14:40.090ZDownstairs last five completed:
208652024-01-24T19:14:40.090ZUpstairs last five completed: 30 29 28 27 26
208662024-01-24T19:14:40.090ZWrote out file "/tmp/test_repair_verify.data"
208672024-01-24T19:14:40.091Z
208682024-01-24T19:14:40.091ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
208692024-01-24T19:14:40.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208702024-01-24T19:14:40.091Z 0 000-019 193 193 193 339 339 339 T F T <---
208712024-01-24T19:14:40.091Z 1 020-039 184 184 184 324 324 324 T F T <---
208722024-01-24T19:14:40.092Z 2 040-059 193 193 193 339 339 339 T F T <---
208732024-01-24T19:14:40.092Z 3 060-079 192 192 192 338 338 338 F F F
208742024-01-24T19:14:40.092Z 4 080-099 192 192 192 338 338 338 T F T <---
208752024-01-24T19:14:40.092Z 5 100-119 193 193 193 339 339 339 T F T <---
208762024-01-24T19:14:40.092Z 6 120-139 190 190 190 337 337 337 T F T <---
208772024-01-24T19:14:40.093Z 7 140-159 192 192 192 338 338 338 F F F
208782024-01-24T19:14:40.093Z 8 160-179 192 192 192 338 338 338 F F F
208792024-01-24T19:14:40.093Z 9 180-199 190 190 190 337 337 337 T F T <---
208802024-01-24T19:14:40.097Z 10 200-219 193 193 193 339 339 339 F F F
208812024-01-24T19:14:40.097Z 11 220-239 192 192 192 338 338 338 F F F
208822024-01-24T19:14:40.097Z 12 240-259 189 189 189 336 336 336 F F F
208832024-01-24T19:14:40.097Z 13 260-279 190 190 190 337 337 337 F F F
208842024-01-24T19:14:40.098Z 14 280-299 189 189 189 336 336 336 F F F
208852024-01-24T19:14:40.098Z 15 300-319 190 190 190 337 337 337 F F F
208862024-01-24T19:14:40.098Z 16 320-339 190 190 190 337 337 337 F F F
208872024-01-24T19:14:40.098Z 17 340-359 185 185 185 326 326 326 F F F
208882024-01-24T19:14:40.099Z 18 360-379 193 193 193 339 339 339 F F F
208892024-01-24T19:14:40.099Z 19 380-399 193 193 193 339 339 339 F F F
208902024-01-24T19:14:40.099Z 20 400-419 194 192 194 340 338 340 F F F <---
208912024-01-24T19:14:40.099Z 21 420-439 190 190 190 337 337 337 T F T <---
208922024-01-24T19:14:40.099Z 22 440-459 190 190 190 337 337 337 F F F
208932024-01-24T19:14:40.100Z 23 460-479 192 192 192 338 338 338 T F T <---
208942024-01-24T19:14:40.100Z 24 480-499 174 174 174 302 302 302 T F T <---
208952024-01-24T19:14:40.100Z 25 500-519 190 190 190 337 337 337 F F F
208962024-01-24T19:14:40.100Z 26 520-539 190 190 190 337 337 337 F F F
208972024-01-24T19:14:40.101Z 27 540-559 189 189 189 336 336 336 F F F
208982024-01-24T19:14:40.101Z 28 560-579 190 190 190 337 337 337 F F F
208992024-01-24T19:14:40.101Z 29 580-599 182 182 182 321 321 321 F F F
209002024-01-24T19:14:40.101ZMax gen: 194, Max flush: 340
209012024-01-24T19:14:40.101ZError: Difference in extent metadata found!
209022024-01-24T19:14:40.102ZA Difference in extent metadata is expected here
209032024-01-24T19:14:40.102ZOn loop 97
209042024-01-24T19:14:40.102Z
209052024-01-24T19:14:40.103ZVerifying data now
209062024-01-24T19:14:41.132ZNo change after verify
209072024-01-24T19:14:41.132ZLoop: 97 Downstairs dump after verify (and repair):
209082024-01-24T19:14:41.160ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209092024-01-24T19:14:41.160Z 0 000-019 195 195 195 341 341 341 F F F
209102024-01-24T19:14:41.160Z 1 020-039 195 195 195 341 341 341 F F F
209112024-01-24T19:14:41.160Z 2 040-059 195 195 195 341 341 341 F F F
209122024-01-24T19:14:41.160Z 3 060-079 192 192 192 338 338 338 F F F
209132024-01-24T19:14:41.160Z 4 080-099 195 195 195 341 341 341 F F F
209142024-01-24T19:14:41.160Z 5 100-119 195 195 195 341 341 341 F F F
209152024-01-24T19:14:41.160Z 6 120-139 195 195 195 341 341 341 F F F
209162024-01-24T19:14:41.160Z 7 140-159 192 192 192 338 338 338 F F F
209172024-01-24T19:14:41.160Z 8 160-179 192 192 192 338 338 338 F F F
209182024-01-24T19:14:41.160Z 9 180-199 195 195 195 341 341 341 F F F
209192024-01-24T19:14:41.160Z 10 200-219 193 193 193 339 339 339 F F F
209202024-01-24T19:14:41.160Z 11 220-239 192 192 192 338 338 338 F F F
209212024-01-24T19:14:41.160Z 12 240-259 189 189 189 336 336 336 F F F
209222024-01-24T19:14:41.160Z 13 260-279 190 190 190 337 337 337 F F F
209232024-01-24T19:14:41.160Z 14 280-299 189 189 189 336 336 336 F F F
209242024-01-24T19:14:41.160Z 15 300-319 190 190 190 337 337 337 F F F
209252024-01-24T19:14:41.160Z 16 320-339 190 190 190 337 337 337 F F F
209262024-01-24T19:14:41.161Z 17 340-359 185 185 185 326 326 326 F F F
209272024-01-24T19:14:41.161Z 18 360-379 193 193 193 339 339 339 F F F
209282024-01-24T19:14:41.162Z 19 380-399 193 193 193 339 339 339 F F F
209292024-01-24T19:14:41.162Z 20 400-419 194 194 194 340 340 340 F F F
209302024-01-24T19:14:41.162Z 21 420-439 195 195 195 341 341 341 F F F
209312024-01-24T19:14:41.162Z 22 440-459 190 190 190 337 337 337 F F F
209322024-01-24T19:14:41.162Z 23 460-479 195 195 195 341 341 341 F F F
209332024-01-24T19:14:41.162Z 24 480-499 195 195 195 341 341 341 F F F
209342024-01-24T19:14:41.162Z 25 500-519 190 190 190 337 337 337 F F F
209352024-01-24T19:14:41.162Z 26 520-539 190 190 190 337 337 337 F F F
209362024-01-24T19:14:41.162Z 27 540-559 189 189 189 336 336 336 F F F
209372024-01-24T19:14:41.162Z 28 560-579 190 190 190 337 337 337 F F F
209382024-01-24T19:14:41.162Z 29 580-599 182 182 182 321 321 321 F F F
209392024-01-24T19:14:41.162ZMax gen: 195, Max flush: 341
209402024-01-24T19:14:41.162Z
209412024-01-24T19:14:41.162ZBegin loop 98
209422024-01-24T19:14:41.162ZDownstairs to restart: 0
209432024-01-24T19:14:41.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.838306897Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916}
209442024-01-24T19:14:41.187Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.838860103Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916}
209452024-01-24T19:14:41.187Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.838891051Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916}
209462024-01-24T19:14:41.187Z{"msg":"Crucible 813a99be-d52f-4865-a598-df112a143d6a has session id: 1cebd781-5629-4740-be78-00a664f27198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.838936186Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209472024-01-24T19:14:41.187Z{"msg":"Upstairs opts: Upstairs UUID: 813a99be-d52f-4865-a598-df112a143d6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.838988264Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209482024-01-24T19:14:41.189Z{"msg":"Crucible stats registered with UUID: 813a99be-d52f-4865-a598-df112a143d6a","v":0,"name":"crucible","level":Crucible runtime is spawned
209492024-01-24T19:14:41.189Z30,"time":"2024-01-24T19:14:39.839140412Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209502024-01-24T19:14:41.189Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839228906Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916}
209512024-01-24T19:14:41.189Z{"msg":"813a99be-d52f-4865-a598-df112a143d6a active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839536339Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209522024-01-24T19:14:41.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839612841Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209532024-01-24T19:14:41.189Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.83974795Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209542024-01-24T19:14:41.189Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839795888Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209552024-01-24T19:14:41.190Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839865941Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209562024-01-24T19:14:41.191Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839911648Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209572024-01-24T19:14:41.191Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.839954918Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209582024-01-24T19:14:41.191Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840045343Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209592024-01-24T19:14:41.191Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840095112Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209602024-01-24T19:14:41.191Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840169371Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209612024-01-24T19:14:41.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840217661Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209622024-01-24T19:14:41.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840308235Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209632024-01-24T19:14:41.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.840447124Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209642024-01-24T19:14:41.191Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.841787472Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209652024-01-24T19:14:41.192Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.841882591Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209662024-01-24T19:14:41.192Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.84192489Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209672024-01-24T19:14:41.192Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.841962976Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209682024-01-24T19:14:41.192Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.842958373Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209692024-01-24T19:14:41.192Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.843128639Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209702024-01-24T19:14:41.192Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.843215493Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209712024-01-24T19:14:41.192Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.843263504Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209722024-01-24T19:14:41.192Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844036764Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209732024-01-24T19:14:41.193Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844102542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209742024-01-24T19:14:41.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844130083Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209752024-01-24T19:14:41.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844180676Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209762024-01-24T19:14:41.193Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844206654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209772024-01-24T19:14:41.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844254805Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209782024-01-24T19:14:41.193Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.844543236Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209792024-01-24T19:14:41.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.84523861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209802024-01-24T19:14:41.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845287556Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209812024-01-24T19:14:41.193Z{"msg":"[0]R flush_numbers[0..12]: [341, 341, 341, 338, 341, 341, 341, 338, 338, 341, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845312058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209822024-01-24T19:14:41.193Z{"msg":"[0]R generation[0..12]: [195, 195, 195, 192, 195, 195, 195, 192, 192, 195, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845382726Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209832024-01-24T19:14:41.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845410646Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209842024-01-24T19:14:41.193Z{"msg":"[1]R flush_numbers[0..12]: [341, 341, 341, 338, 341, 341, 341, 338, 338, 341, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845456117Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209852024-01-24T19:14:41.194Z{"msg":"[1]R generation[0..12]: [195, 195, 195, 192, 195, 195, 195, 192, 192, 195, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845482685Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209862024-01-24T19:14:41.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845527167Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
209872024-01-24T19:14:41.194Z1cebd781-5629-4740-be78-00a664f27198"}
209882024-01-24T19:14:41.194Z{"msg":"[2]R flush_numbers[0..12]: [341, 341, 341, 338, 341, 341, 341, 338, 338, 341, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.8455581Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209892024-01-24T19:14:41.194Z{"msg":"[2]R generation[0..12]: [195, 195, 195, 192, 195, 195, 195, 192, 192, 195, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845602608Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209902024-01-24T19:14:41.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845629706Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209912024-01-24T19:14:41.194Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845672177Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209922024-01-24T19:14:41.194Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845697837Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209932024-01-24T19:14:41.194Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.84573967Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209942024-01-24T19:14:41.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845764759Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209952024-01-24T19:14:41.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845787059Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209962024-01-24T19:14:41.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845807486Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209972024-01-24T19:14:41.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845850484Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209982024-01-24T19:14:41.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845875557Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"1cebd781-5629-4740-be78-00a664f27198"}
209992024-01-24T19:14:41.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845897659Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
210002024-01-24T19:14:41.195Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845943397Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
210012024-01-24T19:14:41.195Z{"msg":"813a99be-d52f-4865-a598-df112a143d6a is now active with session: 1cebd781-5629-4740-be78-00a664f27198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845968491Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
210022024-01-24T19:14:41.195Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.845988926Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916,"session_id":"1cebd781-5629-4740-be78-00a664f27198"}
210032024-01-24T19:14:41.195Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:39.846030932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1916}
210042024-01-24T19:14:41.195ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210052024-01-24T19:14:41.195ZLoading write count information from file "/tmp/test_repair_verify.data"
210062024-01-24T19:14:41.195ZRun Repair workload
210072024-01-24T19:14:41.196Z01/30 Write block 498 len 40960 data: 17 15 13 14 14 10 14 13 11 14
210082024-01-24T19:14:41.196Z02/30 Read block 377 len 8192
210092024-01-24T19:14:41.208Z03/30 Read block 411 len 24576
210102024-01-24T19:14:41.208Z04/30 Write block 113 len 12288 data: 15 10 10
210112024-01-24T19:14:41.208Z05/30 Write block 508 len 8192 data: 13 14
210122024-01-24T19:14:41.209Z06/30 Read block 289 len 4096
210132024-01-24T19:14:41.209Z07/30 Flush
210142024-01-24T19:14:41.209Z08/30 Write block 333 len 12288 data: 12 12 13
210152024-01-24T19:14:41.209Z09/30 Read block 161 len 24576
210162024-01-24T19:14:41.209Z10/30 Write block 534 len 36864 data: 15 15 14 18 16 16 14 15 17
210172024-01-24T19:14:41.209Z11/30 Flush
210182024-01-24T19:14:41.209Z12/30 Write block 190 len 28672 data: 13 13 14 15 16 14 15
210192024-01-24T19:14:41.209Z13/30 Write block 398 len 32768 data: 12 14 15 17 16 10 10 13
210202024-01-24T19:14:41.209Z14/30 Read block 338 len 40960
210212024-01-24T19:14:41.209Z15/30 Read block 362 len 32768
210222024-01-24T19:14:41.209Z16/30 Read block 21 len 16384
210232024-01-24T19:14:41.209Z17/30 Write block 552 len 12288 data: 12 13 16
210242024-01-24T19:14:41.209Z18/30 Read block 408 len 20480
210252024-01-24T19:14:41.209Z19/30 Write block 285 len 20480 data: 17 15 15 18 17
210262024-01-24T19:14:41.209Z20/30 Read block 510 len 36864
210272024-01-24T19:14:41.209Z21/30 Write block 86 len 20480 data: 15 12 10 9 9
210282024-01-24T19:14:41.209Z22/30 Write block 548 len 40960 data: 13 12 12 13 13 14 17 15 16 18
210292024-01-24T19:14:41.222Z23/30 Write block 112 len 24576 data: 17 16 11 11 10 8
210302024-01-24T19:14:41.222Z24/30 Write block 3 len 36864 data: 11 11 11 13 14 16 13 12 11
210312024-01-24T19:14:41.222Z25/30 Flush
210322024-01-24T19:14:41.222Z26/30 Write block 130 len 24576 data: 14 15 15 14 18 18
210332024-01-24T19:14:41.224Z27/30 Read block 96 len 8192
210342024-01-24T19:14:41.224Z28/30 Read block 374 len 28672
210352024-01-24T19:14:41.224Z29/30 Write block 266 len 40960 data: 13 15 13 14 15 16 22 23 18 18
210362024-01-24T19:14:41.224Z30/30 Read block 380 len 4096
210372024-01-24T19:14:41.224Z----------------------------------------------------------------
210382024-01-24T19:14:41.224Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:23
210392024-01-24T19:14:41.224ZStates: Active Active Active
210402024-01-24T19:14:41.224ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210412024-01-24T19:14:41.224Z 8 Acked 1007 Write 3 Done Done Done false
210422024-01-24T19:14:41.224Z 9 Acked 1008 Read 6 Done Done Done false
210432024-01-24T19:14:41.224Z 10 Acked 1009 Write 9 Done Done Done false
210442024-01-24T19:14:41.225Z 11 Acked 1010 Flush 0 Sent Done Done false
210452024-01-24T19:14:41.225Z 12 Acked 1011 Write 7 Sent Done Done false
210462024-01-24T19:14:41.225Z 13 Acked 1012 Write 8 Sent Done Done false
210472024-01-24T19:14:41.225Z 14 Acked 1013 Read 10 Sent Done Done false
210482024-01-24T19:14:41.225Z 15 Acked 1014 Read 8 Sent Done Done false
210492024-01-24T19:14:41.236Z 16 Acked 1015 Read 4 Sent Done Done false
210502024-01-24T19:14:41.237Z 17 Acked 1016 Write 3 Sent Done Done false
210512024-01-24T19:14:41.237Z 18 Acked 1017 Read 5 Sent Done Done false
210522024-01-24T19:14:41.237Z 19 Acked 1018 Write 5 Sent Done Done false
210532024-01-24T19:14:41.237Z 20 Acked 1019 Read 9 Sent Done Done false
210542024-01-24T19:14:41.237Z 21 Acked 1020 Write 5 Sent Done Done false
210552024-01-24T19:14:41.237Z 22 Acked 1021 Write 10 Sent Done Done false
210562024-01-24T19:14:41.237Z 23 Acked 1022 Write 6 Sent Done Done false
210572024-01-24T19:14:41.237Z 24 Acked 1023 Write 9 Sent Done Done false
210582024-01-24T19:14:41.238Z 25 Acked 1024 Flush 0 Sent Done Done false
210592024-01-24T19:14:41.238Z 26 Acked 1025 Write 6 Sent Done Done false
210602024-01-24T19:14:41.238Z 27 Acked 1026 Read 2 Sent Done Done false
210612024-01-24T19:14:41.238Z 28 Acked 1027 Read 7 Sent Done Done false
210622024-01-24T19:14:41.238Z 29 Acked 1028 Write 10 Sent Done Done false
210632024-01-24T19:14:41.238Z 30 Acked 1029 Read 1 Sent Done Done false
210642024-01-24T19:14:41.238Z STATES DS:0 DS:1 DS:2 TOTAL
210652024-01-24T19:14:41.238Z New 0 0 0 0
210662024-01-24T19:14:41.238Z Sent 20 0 0 20
210672024-01-24T19:14:41.238Z Done 3 23 23 49
210682024-01-24T19:14:41.238Z Skipped 0 0 0 0
210692024-01-24T19:14:41.238Z Error 0 0 0 0
210702024-01-24T19:14:41.238ZLast Flush: 1006 1024 1024
210712024-01-24T19:14:41.238ZDownstairs last five completed: 1006 1005 1004 1003 1002
210722024-01-24T19:14:41.238ZUpstairs last five completed: 30 29 28 27 26
210732024-01-24T19:14:41.238ZWrote out file "/tmp/test_repair_verify.data"
210742024-01-24T19:14:41.238Z
210752024-01-24T19:14:41.253ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
210762024-01-24T19:14:41.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210772024-01-24T19:14:41.260Z 0 000-019 195 196 196 341 344 344 F F F <---
210782024-01-24T19:14:41.268Z 1 020-039 195 195 195 341 341 341 F F F
210792024-01-24T19:14:41.268Z 2 040-059 195 195 195 341 341 341 F F F
210802024-01-24T19:14:41.268Z 3 060-079 192 192 192 338 338 338 F F F
210812024-01-24T19:14:41.269Z 4 080-099 195 196 196 341 344 344 F F F <---
210822024-01-24T19:14:41.269Z 5 100-119 196 196 196 342 344 344 F F F <---
210832024-01-24T19:14:41.269Z 6 120-139 195 195 195 341 341 341 F T T <---
210842024-01-24T19:14:41.270Z 7 140-159 192 192 192 338 338 338 F F F
210852024-01-24T19:14:41.270Z 8 160-179 192 192 192 338 338 338 F F F
210862024-01-24T19:14:41.270Z 9 180-199 195 196 196 341 344 344 F F F <---
210872024-01-24T19:14:41.270Z 10 200-219 193 193 193 339 339 339 F F F
210882024-01-24T19:14:41.270Z 11 220-239 192 192 192 338 338 338 F F F
210892024-01-24T19:14:41.270Z 12 240-259 189 189 189 336 336 336 F F F
210902024-01-24T19:14:41.270Z 13 260-279 190 190 190 337 337 337 F T T <---
210912024-01-24T19:14:41.270Z 14 280-299 189 196 196 336 344 344 F F F <---
210922024-01-24T19:14:41.270Z 15 300-319 190 190 190 337 337 337 F F F
210932024-01-24T19:14:41.271Z 16 320-339 190 196 196 337 343 343 T F F <---
210942024-01-24T19:14:41.271Z 17 340-359 185 185 185 326 326 326 F F F
210952024-01-24T19:14:41.271Z 18 360-379 193 193 193 339 339 339 F F F
210962024-01-24T19:14:41.271Z 19 380-399 193 196 196 339 344 344 F F F <---
210972024-01-24T19:14:41.271Z 20 400-419 194 196 196 340 344 344 F F F <---
210982024-01-24T19:14:41.271Z 21 420-439 195 195 195 341 341 341 F F F
210992024-01-24T19:14:41.271Z 22 440-459 190 190 190 337 337 337 F F F
211002024-01-24T19:14:41.271Z 23 460-479 195 195 195 341 341 341 F F F
211012024-01-24T19:14:41.271Z 24 480-499 196 196 196 342 342 342 F F F
211022024-01-24T19:14:41.271Z 25 500-519 196 196 196 342 342 342 F F F
211032024-01-24T19:14:41.271Z 26 520-539 190 196 196 337 343 343 T F F <---
211042024-01-24T19:14:41.271Z 27 540-559 189 196 196 336 344 344 T F F <---
211052024-01-24T19:14:41.271Z 28 560-579 190 190 190 337 337 337 F F F
211062024-01-24T19:14:41.271Z 29 580-599 182 182 182 321 321 321 F F F
211072024-01-24T19:14:41.271ZMax gen: 196, Max flush: 344
211082024-01-24T19:14:41.272ZError: Difference in extent metadata found!
211092024-01-24T19:14:41.272ZA Difference in extent metadata is expected here
211102024-01-24T19:14:41.272ZOn loop 98
211112024-01-24T19:14:41.272Z
211122024-01-24T19:14:41.280ZVerifying data now
211132024-01-24T19:14:47.371ZNo change after verify
211142024-01-24T19:14:47.372ZLoop: 98 Downstairs dump after verify (and repair):
211152024-01-24T19:14:47.386ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211162024-01-24T19:14:47.386Z 0 000-019 196 196 196 344 344 344 F F F
211172024-01-24T19:14:47.386Z 1 020-039 195 195 195 341 341 341 F F F
211182024-01-24T19:14:47.386Z 2 040-059 195 195 195 341 341 341 F F F
211192024-01-24T19:14:47.386Z 3 060-079 192 192 192 338 338 338 F F F
211202024-01-24T19:14:47.386Z 4 080-099 196 196 196 344 344 344 F F F
211212024-01-24T19:14:47.390Z 5 100-119 196 196 196 344 344 344 F F F
211222024-01-24T19:14:47.390Z 6 120-139 197 197 197 345 345 345 F F F
211232024-01-24T19:14:47.390Z 7 140-159 192 192 192 338 338 338 F F F
211242024-01-24T19:14:47.394Z 8 160-179 192 192 192 338 338 338 F F F
211252024-01-24T19:14:47.395Z 9 180-199 196 196 196 344 344 344 F F F
211262024-01-24T19:14:47.395Z 10 200-219 193 193 193 339 339 339 F F F
211272024-01-24T19:14:47.395Z 11 220-239 192 192 192 338 338 338 F F F
211282024-01-24T19:14:47.395Z 12 240-259 189 189 189 336 336 336 F F F
211292024-01-24T19:14:47.395Z 13 260-279 197 197 197 345 345 345 F F F
211302024-01-24T19:14:47.395Z 14 280-299 196 196 196 344 344 344 F F F
211312024-01-24T19:14:47.395Z 15 300-319 190 190 190 337 337 337 F F F
211322024-01-24T19:14:47.395Z 16 320-339 196 196 196 343 343 343 F F F
211332024-01-24T19:14:47.395Z 17 340-359 185 185 185 326 326 326 F F F
211342024-01-24T19:14:47.396Z 18 360-379 193 193 193 339 339 339 F F F
211352024-01-24T19:14:47.396Z 19 380-399 196 196 196 344 344 344 F F F
211362024-01-24T19:14:47.396Z 20 400-419 196 196 196 344 344 344 F F F
211372024-01-24T19:14:47.396Z 21 420-439 195 195 195 341 341 341 F F F
211382024-01-24T19:14:47.396Z 22 440-459 190 190 190 337 337 337 F F F
211392024-01-24T19:14:47.396Z 23 460-479 195 195 195 341 341 341 F F F
211402024-01-24T19:14:47.398Z 24 480-499 196 196 196 342 342 342 F F F
211412024-01-24T19:14:47.398Z 25 500-519 196 196 196 342 342 342 F F F
211422024-01-24T19:14:47.398Z 26 520-539 196 196 196 343 343 343 F F F
211432024-01-24T19:14:47.398Z 27 540-559 196 196 196 344 344 344 F F F
211442024-01-24T19:14:47.398Z 28 560-579 190 190 190 337 337 337 F F F
211452024-01-24T19:14:47.398Z 29 580-599 182 182 182 321 321 321 F F F
211462024-01-24T19:14:47.398ZMax gen: 197, Max flush: 345
211472024-01-24T19:14:47.398Z
211482024-01-24T19:14:47.398ZBegin loop 99
211492024-01-24T19:14:47.398ZDownstairs to restart: 2
211502024-01-24T19:14:47.449Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.094584045Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924}
211512024-01-24T19:14:47.450Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9ba5d9a1454daf70384f9d8841ab0e9477eac04e\",\n git_commit_timestamp: \"2024-01-24T18:28:36.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095083861Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924}
211522024-01-24T19:14:47.450Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095115125Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924}
211532024-01-24T19:14:47.450Z{"msg":"Crucible 33af739f-5f2b-4a09-9e30-3c45bb3ecee0 has session id: cbff0da2-602c-4af0-8fe3-ed75a228b5be","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095136561Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211542024-01-24T19:14:47.452Z{"msg":"Upstairs opts: Upstairs UUID: 33af739f-5f2b-4a09-9e30-3c45bb3ecee0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095170549Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211552024-01-24T19:14:47.460Z{"msg":"Crucible stats registered with UUID: 33af739f-5f2b-4a09-9e30-3c45bb3ecee0","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095195932Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211562024-01-24T19:14:47.460ZCrucible runtime is spawned
211572024-01-24T19:14:47.460Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095670842Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924}
211582024-01-24T19:14:47.460Z{"msg":"33af739f-5f2b-4a09-9e30-3c45bb3ecee0 active request set","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095793058Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211592024-01-24T19:14:47.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.095951242Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211602024-01-24T19:14:47.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096059285Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211612024-01-24T19:14:47.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096105742Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211622024-01-24T19:14:47.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096146807Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211632024-01-24T19:14:47.460Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.09643543Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211642024-01-24T19:14:47.460Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096541153Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211652024-01-24T19:14:47.460Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096681082Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211662024-01-24T19:14:47.460Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096793188Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211672024-01-24T19:14:47.460Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096840875Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211682024-01-24T19:14:47.460Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096908667Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211692024-01-24T19:14:47.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096954189Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211702024-01-24T19:14:47.461Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.096986095Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211712024-01-24T19:14:47.461Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097448496Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211722024-01-24T19:14:47.461Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097532228Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211732024-01-24T19:14:47.461Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097560719Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211742024-01-24T19:14:47.464Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097588056Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211752024-01-24T19:14:47.464Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097845366Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211762024-01-24T19:14:47.464Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.097884916Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211772024-01-24T19:14:47.464Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.098649224Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211782024-01-24T19:14:47.464Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.098716347Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211792024-01-24T19:14:47.464Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.098860825Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211802024-01-24T19:14:47.464Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099550034Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211812024-01-24T19:14:47.464Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099646905Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211822024-01-24T19:14:47.464Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099678592Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211832024-01-24T19:14:47.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099703314Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211842024-01-24T19:14:47.465Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099769418Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211852024-01-24T19:14:47.465Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.09979905Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211862024-01-24T19:14:47.465Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099915093Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211872024-01-24T19:14:47.465Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.099972654Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211882024-01-24T19:14:47.466Z{"msg":"[0]R flush_numbers[0..12]: [344, 341, 341, 338, 344, 344, 345, 338, 338, 344, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100002569Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211892024-01-24T19:14:47.466Z{"msg":"[0]R generation[0..12]: [196, 195, 195, 192, 196, 196, 197, 192, 192, 196, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100027159Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211902024-01-24T19:14:47.466Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100052887Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211912024-01-24T19:14:47.466Z{"msg":"[1]R flush_numbers[0..12]: [344, 341, 341, 338, 344, 344, 345, 338, 338, 344, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100076868Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211922024-01-24T19:14:47.466Z{"msg":"[1]R generation[0..12]: [196, 195, 195, 192, 196, 196, 197, 192, 192, 196, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100101413Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211932024-01-24T19:14:47.470Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100125607Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211942024-01-24T19:14:47.470Z{"msg":"[2]R flush_numbers[0..12]: [344, 341, 341, 338, 344, 344, 345, 338, 338, 344, 339, 338]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100170893Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211952024-01-24T19:14:47.470Z{"msg":"[2]R generation[0..12]: [196, 195, 195, 192, 196, 196, 197, 192, 192, 196, 193, 192]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100197642Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211962024-01-24T19:14:47.470Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.10024233Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211972024-01-24T19:14:47.470Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100268885Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
211982024-01-24T19:14:47.470Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100291542Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
211992024-01-24T19:14:47.470Zsession_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212002024-01-24T19:14:47.470Z{"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100380934Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212012024-01-24T19:14:47.471Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100432168Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212022024-01-24T19:14:47.471Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.10045907Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212032024-01-24T19:14:47.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100480053Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212042024-01-24T19:14:47.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100523753Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212052024-01-24T19:14:47.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100549017Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212062024-01-24T19:14:47.475Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100597973Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212072024-01-24T19:14:47.476Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100622144Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212082024-01-24T19:14:47.476Z{"msg":"33af739f-5f2b-4a09-9e30-3c45bb3ecee0 is now active with session: cbff0da2-602c-4af0-8fe3-ed75a228b5be","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100643954Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212092024-01-24T19:14:47.476Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100710385Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924,"session_id":"cbff0da2-602c-4af0-8fe3-ed75a228b5be"}
212102024-01-24T19:14:47.476Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-01-24T19:14:46.100734035Z","hostname":"ip-10-150-1-18.us-west-2.compute.internal","pid":1924}
212112024-01-24T19:14:47.476ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212122024-01-24T19:14:47.476ZLoading write count information from file "/tmp/test_repair_verify.data"
212132024-01-24T19:14:47.476ZRun Repair workload
212142024-01-24T19:14:47.476Z01/30 Write block 206 len 4096 data: 9
212152024-01-24T19:14:47.476Z02/30 Write block 432 len 20480 data: 16 17 18 13 13
212162024-01-24T19:14:47.499Z03/30 Read block 483 len 12288
212172024-01-24T19:14:47.499Z04/30 Read block 374 len 16384
212182024-01-24T19:14:47.500Z05/30 Write block 401 len 24576 data: 18 17 11 11 14 16
212192024-01-24T19:14:47.500Z06/30 Write block 85 len 36864 data: 14 16 13 11 10 10 11 11 12
212202024-01-24T19:14:47.500Z07/30 Flush
212212024-01-24T19:14:47.500Z08/30 Write block 51 len 8192 data: 17 17
212222024-01-24T19:14:47.500Z09/30 Write block 25 len 24576 data: 11 13 15 18 18 16
212232024-01-24T19:14:47.500Z10/30 Read block 97 len 36864
212242024-01-24T19:14:47.500Z11/30 Read block 470 len 12288
212252024-01-24T19:14:47.500Z12/30 Write block 489 len 36864 data: 11 11 10 12 13 13 12 16 16
212262024-01-24T19:14:47.500Z13/30 Flush
212272024-01-24T19:14:47.500Z14/30 Write block 359 len 12288 data: 13 13 18
212282024-01-24T19:14:47.500Z15/30 Read block 110 len 8192
212292024-01-24T19:14:47.500Z16/30 Write block 204 len 12288 data: 13 12 10
212302024-01-24T19:14:47.501Z17/30 Read block 438 len 8192
212312024-01-24T19:14:47.501Z18/30 Read block 89 len 32768
212322024-01-24T19:14:47.501Z19/30 Read block 143 len 4096
212332024-01-24T19:14:47.501Z20/30 Write block 238 len 16384 data: 13 14 13 12
212342024-01-24T19:14:47.501Z21/30 Write block 107 len 24576 data: 24 23 24 23 21 18
212352024-01-24T19:14:47.501Z22/30 Read block 337 len 28672
212362024-01-24T19:14:47.515Z23/30 Write block 531 len 32768 data: 15 11 14 16 16 15 19 17
212372024-01-24T19:14:47.515Z24/30 Read block 323 len 8192
212382024-01-24T19:14:47.515Z25/30 Write block 365 len 36864 data: 19 19 20 18 17 16 16 15 17
212392024-01-24T19:14:47.515Z26/30 Read block 208 len 16384
212402024-01-24T19:14:47.515Z27/30 Write block 175 len 8192 data: 15 13
212412024-01-24T19:14:47.515Z28/30 Write block 96 len 12288 data: 12 14 13
212422024-01-24T19:14:47.519Z29/30 Read block 477 len 32768
212432024-01-24T19:14:47.519Z30/30 Write block 582 len 32768 data: 11 15 16 15 16 16 15 13
212442024-01-24T19:14:47.519Z----------------------------------------------------------------
212452024-01-24T19:14:47.520Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
212462024-01-24T19:14:47.520ZStates: Active Active Active
212472024-01-24T19:14:47.520ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
212482024-01-24T19:14:47.520Z 1 Acked 1000 Write 1 Done Done Done false
212492024-01-24T19:14:47.520Z 2 Acked 1001 Write 5 Done Done Sent false
212502024-01-24T19:14:47.520Z 3 Acked 1002 Read 3 Done Done Sent false
212512024-01-24T19:14:47.520Z 4 Acked 1003 Read 4 Done Done Sent false
212522024-01-24T19:14:47.520Z 5 Acked 1004 Write 6 Done Done Sent false
212532024-01-24T19:14:47.520Z 6 Acked 1005 Write 9 Done Done Sent false
212542024-01-24T19:14:47.520Z 7 Acked 1006 Flush 0 Done Done Sent false
212552024-01-24T19:14:47.520Z 8 Acked 1007 Write 2 Done Done Sent false
212562024-01-24T19:14:47.520Z 9 Acked 1008 Write 6 Done Done Sent false
212572024-01-24T19:14:47.520Z 10 Acked 1009 Read 9 Done Done Sent false
212582024-01-24T19:14:47.521Z 11 Acked 1010 Read 3 Done Done Sent false
212592024-01-24T19:14:47.521Z 12 Acked 1011 Write 9 Done Done Sent false
212602024-01-24T19:14:47.521Z 13 Acked 1012 Flush 0 Done Done Sent false
212612024-01-24T19:14:47.521Z 14 Acked 1013 Write 3 Done Done Sent false
212622024-01-24T19:14:47.521Z 15 Acked 1014 Read 2 Done Done Sent false
212632024-01-24T19:14:47.521Z 16 Acked 1015 Write 3 Done Done Sent false
212642024-01-24T19:14:47.521Z 17 Acked 1016 Read 2 Done Done Sent false
212652024-01-24T19:14:47.522Z 18 Acked 1017 Read 8 Done Done Sent false
212662024-01-24T19:14:47.522Z 19 Acked 1018 Read 1 Done Done Sent false
212672024-01-24T19:14:47.522Z 20 Acked 1019 Write 4 Done Done Sent false
212682024-01-24T19:14:47.523Z 21 Acked 1020 Write 6 Done Done Sent false
212692024-01-24T19:14:47.523Z 22 Acked 1021 Read 7 Done Done Sent false
212702024-01-24T19:14:47.523Z 23 Acked 1022 Write 8 Done Done Sent false
212712024-01-24T19:14:47.523Z 24 Acked 1023 Read 2 Done Done Sent false
212722024-01-24T19:14:47.523Z 25 Acked 1024 Write 9 Done Done Sent false
212732024-01-24T19:14:47.523Z 26 Acked 1025 Read 4 Done Done Sent false
212742024-01-24T19:14:47.523Z 27 Acked 1026 Write 2 Done Done Sent false
212752024-01-24T19:14:47.523Z 28 Acked 1027 Write 3 Done Done Sent false
212762024-01-24T19:14:47.523Z 29 Acked 1028 Read 8 Done Done Sent false
212772024-01-24T19:14:47.523Z 30 Acked 1029 Write 8 Sent Sent Sent false
212782024-01-24T19:14:47.523Z STATES DS:0 DS:1 DS:2 TOTAL
212792024-01-24T19:14:47.523Z New 0 0 0 0
212802024-01-24T19:14:47.524Z Sent 1 1 29 31
212812024-01-24T19:14:47.524Z Done 29 29 1 59
212822024-01-24T19:14:47.524Z Skipped 0 0 0 0
212832024-01-24T19:14:47.524Z Error 0 0 0 0
212842024-01-24T19:14:47.524ZLast Flush: 1012 1012 0
212852024-01-24T19:14:47.524ZDownstairs last five completed:
212862024-01-24T19:14:47.524ZUpstairs last five completed: 30 29 28 27 26
212872024-01-24T19:14:47.524ZWrote out file "/tmp/test_repair_verify.data"
212882024-01-24T19:14:47.524Z
212892024-01-24T19:14:47.524ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
212902024-01-24T19:14:47.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212912024-01-24T19:14:47.525Z 0 000-019 196 196 196 344 344 344 F F F
212922024-01-24T19:14:47.525Z 1 020-039 198 198 195 347 347 341 F F F <---
212932024-01-24T19:14:47.525Z 2 040-059 198 198 195 347 347 341 F F F <---
212942024-01-24T19:14:47.525Z 3 060-079 192 192 192 338 338 338 F F F
212952024-01-24T19:14:47.525Z 4 080-099 198 198 196 346 346 344 T T F <---
212962024-01-24T19:14:47.525Z 5 100-119 196 196 196 344 344 344 T T F <---
212972024-01-24T19:14:47.525Z 6 120-139 197 197 197 345 345 345 F F F
212982024-01-24T19:14:47.525Z 7 140-159 192 192 192 338 338 338 F F F
212992024-01-24T19:14:47.526Z 8 160-179 192 192 192 338 338 338 T T F <---
213002024-01-24T19:14:47.526Z 9 180-199 196 196 196 344 344 344 F F F
213012024-01-24T19:14:47.526Z 10 200-219 198 198 193 346 346 339 T T T <---
213022024-01-24T19:14:47.526Z 11 220-239 192 192 192 338 338 338 T T F <---
213032024-01-24T19:14:47.526Z 12 240-259 189 189 189 336 336 336 T T F <---
213042024-01-24T19:14:47.526Z 13 260-279 197 197 197 345 345 345 F F F
213052024-01-24T19:14:47.526Z 14 280-299 196 196 196 344 344 344 F F F
213062024-01-24T19:14:47.526Z 15 300-319 190 190 190 337 337 337 F F F
213072024-01-24T19:14:47.526Z 16 320-339 196 196 196 343 343 343 F F F
213082024-01-24T19:14:47.526Z 17 340-359 185 185 185 326 326 326 T T F <---
213092024-01-24T19:14:47.526Z 18 360-379 193 193 193 339 339 339 T T F <---
213102024-01-24T19:14:47.526Z 19 380-399 196 196 196 344 344 344 F F F
213112024-01-24T19:14:47.526Z 20 400-419 198 198 196 346 346 344 F F F <---
213122024-01-24T19:14:47.526Z 21 420-439 198 198 195 346 346 341 F F F <---
213132024-01-24T19:14:47.534Z 22 440-459 190 190 190 337 337 337 F F F
213142024-01-24T19:14:47.534Z 23 460-479 195 195 195 341 341 341 F F F
213152024-01-24T19:14:47.534Z 24 480-499 198 198 196 347 347 342 F F F <---
213162024-01-24T19:14:47.534Z 25 500-519 196 196 196 342 342 342 F F F
213172024-01-24T19:14:47.534Z 26 520-539 196 196 196 343 343 343 T T F <---
213182024-01-24T19:14:47.534Z 27 540-559 196 196 196 344 344 344 F F F
213192024-01-24T19:14:47.534Z 28 560-579 190 190 190 337 337 337 F F F
213202024-01-24T19:14:47.534Z 29 580-599 182 182 182 321 321 321 T T F <---
213212024-01-24T19:14:47.534ZMax gen: 198, Max flush: 347
213222024-01-24T19:14:47.534ZError: Difference in extent metadata found!
213232024-01-24T19:14:47.535ZA Difference in extent metadata is expected here
213242024-01-24T19:14:47.535ZOn loop 99
213252024-01-24T19:14:47.535Z
213262024-01-24T19:14:47.539ZVerifying data now
213272024-01-24T19:14:52.834ZNo change after verify
213282024-01-24T19:14:52.835ZLoop: 99 Downstairs dump after verify (and repair):
213292024-01-24T19:14:52.859ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213302024-01-24T19:14:52.859Z 0 000-019 196 196 196 344 344 344 F F F
213312024-01-24T19:14:52.859Z 1 020-039 198 198 198 347 347 347 F F F
213322024-01-24T19:14:52.866Z 2 040-059 198 198 198 347 347 347 F F F
213332024-01-24T19:14:52.866Z 3 060-079 192 192 192 338 338 338 F F F
213342024-01-24T19:14:52.867Z 4 080-099 199 199 199 348 348 348 F F F
213352024-01-24T19:14:52.867Z 5 100-119 199 199 199 348 348 348 F F F
213362024-01-24T19:14:52.867Z 6 120-139 197 197 197 345 345 345 F F F
213372024-01-24T19:14:52.867Z 7 140-159 192 192 192 338 338 338 F F F
213382024-01-24T19:14:52.867Z 8 160-179 199 199 199 348 348 348 F F F
213392024-01-24T19:14:52.868Z 9 180-199 196 196 196 344 344 344 F F F
213402024-01-24T19:14:52.874Z 10 200-219 199 199 199 348 348 348 F F F
213412024-01-24T19:14:52.874Z 11 220-239 199 199 199 348 348 348 F F F
213422024-01-24T19:14:52.874Z 12 240-259 199 199 199 348 348 348 F F F
213432024-01-24T19:14:52.875Z 13 260-279 197 197 197 345 345 345 F F F
213442024-01-24T19:14:52.875Z 14 280-299 196 196 196 344 344 344 F F F
213452024-01-24T19:14:52.875Z 15 300-319 190 190 190 337 337 337 F F F
213462024-01-24T19:14:52.876Z 16 320-339 196 196 196 343 343 343 F F F
213472024-01-24T19:14:52.876Z 17 340-359 199 199 199 348 348 348 F F F
213482024-01-24T19:14:52.876Z 18 360-379 199 199 199 348 348 348 F F F
213492024-01-24T19:14:52.876Z 19 380-399 196 196 196 344 344 344 F F F
213502024-01-24T19:14:52.877Z 20 400-419 198 198 198 346 346 346 F F F
213512024-01-24T19:14:52.877Z 21 420-439 198 198 198 346 346 346 F F F
213522024-01-24T19:14:52.877Z 22 440-459 190 190 190 337 337 337 F F F
213532024-01-24T19:14:52.877Z 23 460-479 195 195 195 341 341 341 F F F
213542024-01-24T19:14:52.878Z 24 480-499 198 198 198 347 347 347 F F F
213552024-01-24T19:14:52.878Z 25 500-519 196 196 196 342 342 342 F F F
213562024-01-24T19:14:52.878Z 26 520-539 199 199 199 348 348 348 F F F
213572024-01-24T19:14:52.878Z 27 540-559 196 196 196 344 344 344 F F F
213582024-01-24T19:14:52.878Z 28 560-579 190 190 190 337 337 337 F F F
213592024-01-24T19:14:52.878Z 29 580-599 199 199 199 348 348 348 F F F
213602024-01-24T19:14:52.879ZMax gen: 199, Max flush: 348
213612024-01-24T19:14:52.879Z9:30 Test duration
213622024-01-24T19:14:52.879ZTest completed
213632024-01-24T19:14:52.879Z
213642024-01-24T19:14:52.879Zreal 9:29.685003168
213652024-01-24T19:14:52.880Zuser 2:47.769824440
213662024-01-24T19:14:52.880Zsys 28.391345982
213672024-01-24T19:14:52.880Ztrap 0.141826721
213682024-01-24T19:14:52.880Ztflt 0.095669704
213692024-01-24T19:14:52.881Zdflt 0.125208684
213702024-01-24T19:14:52.881Zkflt 0.067826005
213712024-01-24T19:14:52.881Zlock 7:26:54.622111358
213722024-01-24T19:14:52.881Zslp 44:59.853698162
213732024-01-24T19:14:52.881Zlat 10.257992746
213742024-01-24T19:14:52.881Zstop 0.065362335
213752024-01-24T19:14:52.882Zprocess exited: duration 582664 ms, exit code 0
 
213762024-01-24T19:14:52.901Zfound 4 output files
213772024-01-24T19:14:52.901Zuploading: /tmp/test_repair_ds8810.txt (49274 bytes)
213782024-01-24T19:14:53.950Zuploaded: /tmp/test_repair_ds8810.txt
213792024-01-24T19:14:53.951Zuploading: /tmp/test_repair_ds8820.txt (54072 bytes)
213802024-01-24T19:14:55.103Zuploaded: /tmp/test_repair_ds8820.txt
213812024-01-24T19:14:55.107Zuploading: /tmp/test_repair_ds8830.txt (23223 bytes)
213822024-01-24T19:14:56.129Zuploaded: /tmp/test_repair_ds8830.txt
213832024-01-24T19:14:56.129Zuploading: /tmp/test_repair_out.txt (97571 bytes)
213842024-01-24T19:14:57.172Zuploaded: /tmp/test_repair_out.txt