01HPJ5CAW0MJ3W0W506046QXC3: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HPJ5CQR5GAAR0AAXV51M8N8N

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-13T21:35:54.446Zjob dependencies complete; ready to run (waiting for 16 m 54 s)
22024-02-13T21:36:59.130Zjob assigned to worker 01HPJ6C2F6YBRT9Q7PSNKGGBKB (queued for 1 m 4 s)
32024-02-13T21:36:59.681Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-13T21:37:22.169Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-13T21:37:22.169Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-13T21:37:39.616Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-13T21:37:39.616Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-13T21:38:12.717Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-13T21:38:12.717Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-13T21:38:32.952Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-13T21:38:32.962Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-13T21:38:32.970Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-13T21:38:32.970Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-13T21:38:32.983Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-13T21:38:32.983Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-13T21:38:34.523Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-13T21:38:34.523Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-13T21:38:34.965Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-13T21:38:34.965Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-13T21:38:35.514Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-13T21:38:35.514Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-13T21:38:35.998Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-13T21:38:35.998Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-13T21:38:36.429Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-13T21:38:36.430Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-13T21:39:05.273Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-13T21:39:05.273Zstarting task 0: "setup"
282024-02-13T21:39:05.525Z++ uname -s
292024-02-13T21:39:05.528Z+ kern=SunOS
302024-02-13T21:39:05.528Z+ case "$kern" in
312024-02-13T21:39:05.528Z+ groupadd -g 12345 build
322024-02-13T21:39:05.528Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-13T21:39:07.541Z+ zfs create -o mountpoint=/work rpool/work
342024-02-13T21:39:07.652Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-13T21:39:07.659Z+ home_fs=zfs
362024-02-13T21:39:07.659Z+ [[ zfs == autofs ]]
372024-02-13T21:39:07.659Z+ mkdir -p /home/build
382024-02-13T21:39:07.660Z+ chown build:build /home/build /work
392024-02-13T21:39:09.658Z+ chmod 0700 /home/build /work
402024-02-13T21:39:09.665Zprocess exited: duration 4409 ms, exit code 0
 
412024-02-13T21:39:09.693Zstarting task 1: "authentication"
422024-02-13T21:39:09.726Zprocess exited: duration 28 ms, exit code 0
 
432024-02-13T21:39:09.770Zstarting task 2: "build"
442024-02-13T21:39:09.779Z+ banner cores
452024-02-13T21:39:09.788Z
462024-02-13T21:39:09.788Z #### #### ##### ###### ####
472024-02-13T21:39:09.788Z # # # # # # # #
482024-02-13T21:39:09.788Z # # # # # ##### ####
492024-02-13T21:39:09.788Z # # # ##### # #
502024-02-13T21:39:09.788Z # # # # # # # # #
512024-02-13T21:39:09.788Z #### #### # # ###### ####
522024-02-13T21:39:09.788Z
532024-02-13T21:39:09.795Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-13T21:39:09.813Z+ echo 'input bins dir contains:'
552024-02-13T21:39:09.813Zinput bins dir contains:
562024-02-13T21:39:09.814Z+ ls -ltr /input/build/work/bins
572024-02-13T21:39:09.814Ztotal 761129
582024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 121565893 Feb 13 21:37 crucible-downstairs.gz
592024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 91138074 Feb 13 21:37 crucible-hammer.gz
602024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 104804038 Feb 13 21:37 crutest.gz
612024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 71941606 Feb 13 21:38 dsc.gz
622024-02-13T21:39:09.814Z+ echo 'input script dir contains:'
632024-02-13T21:39:09.814Zinput script dir contains:
642024-02-13T21:39:09.814Z+ ls -ltr /input/build/work/scripts
652024-02-13T21:39:09.814Ztotal 39
662024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 3056 Feb 13 21:38 perf-downstairs-tick.d
672024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 1738 Feb 13 21:38 test_ds.sh
682024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 3746 Feb 13 21:38 test_live_repair.sh
692024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 6416 Feb 13 21:38 test_repair.sh
702024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 2969 Feb 13 21:38 test_replay.sh
712024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 9465 Feb 13 21:38 test_up.sh
722024-02-13T21:39:09.814Z-rw-r--r-- 1 root root 2824 Feb 13 21:38 upstairs_info.d
732024-02-13T21:39:09.814Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-13T21:39:09.814Z+ echo ' chmod input script dir contains:'
752024-02-13T21:39:09.814Z+ ls -ltr /input/build/work/scripts
762024-02-13T21:39:09.814Z chmod input script dir contains:
772024-02-13T21:39:09.814Ztotal 39
782024-02-13T21:39:09.814Z-rwxr-xr-x 1 root root 3056 Feb 13 21:38 perf-downstairs-tick.d
792024-02-13T21:39:09.814Z-rwxr-xr-x 1 root root 1738 Feb 13 21:38 test_ds.sh
802024-02-13T21:39:09.815Z-rwxr-xr-x 1 root root 3746 Feb 13 21:38 test_live_repair.sh
812024-02-13T21:39:09.815Z-rwxr-xr-x 1 root root 6416 Feb 13 21:38 test_repair.sh
822024-02-13T21:39:09.815Z-rwxr-xr-x 1 root root 2969 Feb 13 21:38 test_replay.sh
832024-02-13T21:39:09.815Z-rwxr-xr-x 1 root root 9465 Feb 13 21:38 test_up.sh
842024-02-13T21:39:09.815Z-rwxr-xr-x 1 root root 2824 Feb 13 21:38 upstairs_info.d
852024-02-13T21:39:09.815Z+ banner unpack
862024-02-13T21:39:09.822Z
872024-02-13T21:39:09.823Z # # # # ##### ## #### # #
882024-02-13T21:39:09.823Z # # ## # # # # # # # # #
892024-02-13T21:39:09.823Z # # # # # # # # # # ####
902024-02-13T21:39:09.823Z # # # # # ##### ###### # # #
912024-02-13T21:39:09.823Z # # # ## # # # # # # #
922024-02-13T21:39:09.823Z #### # # # # # #### # #
932024-02-13T21:39:09.823Z
942024-02-13T21:39:09.823Z+ mkdir -p /var/tmp/bins
952024-02-13T21:39:09.823Z+ for t in "$input/bins/"*.gz
962024-02-13T21:39:09.823Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-13T21:39:09.823Z+ b=crucible-downstairs.gz
982024-02-13T21:39:09.823Z+ b=crucible-downstairs
992024-02-13T21:39:09.823Z+ gunzip
1002024-02-13T21:39:12.650Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-13T21:39:12.660Z+ for t in "$input/bins/"*.gz
1022024-02-13T21:39:12.661Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-13T21:39:12.661Z+ b=crucible-hammer.gz
1042024-02-13T21:39:12.661Z+ b=crucible-hammer
1052024-02-13T21:39:12.661Z+ gunzip
1062024-02-13T21:39:14.709Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-13T21:39:14.720Z+ for t in "$input/bins/"*.gz
1082024-02-13T21:39:14.720Z++ basename /input/build/work/bins/crutest.gz
1092024-02-13T21:39:14.720Z+ b=crutest.gz
1102024-02-13T21:39:14.721Z+ b=crutest
1112024-02-13T21:39:14.721Z+ gunzip
1122024-02-13T21:39:17.142Z+ chmod +x /var/tmp/bins/crutest
1132024-02-13T21:39:17.149Z+ for t in "$input/bins/"*.gz
1142024-02-13T21:39:17.150Z++ basename /input/build/work/bins/dsc.gz
1152024-02-13T21:39:17.150Z+ b=dsc.gz
1162024-02-13T21:39:17.150Z+ b=dsc
1172024-02-13T21:39:17.150Z+ gunzip
1182024-02-13T21:39:18.758Z+ chmod +x /var/tmp/bins/dsc
1192024-02-13T21:39:18.767Z+ export BINDIR=/var/tmp/bins
1202024-02-13T21:39:18.767Z+ BINDIR=/var/tmp/bins
1212024-02-13T21:39:18.767Z+ banner repair
1222024-02-13T21:39:18.767Z
1232024-02-13T21:39:18.767Z ##### ###### ##### ## # #####
1242024-02-13T21:39:18.767Z # # # # # # # # # #
1252024-02-13T21:39:18.767Z # # ##### # # # # # # #
1262024-02-13T21:39:18.767Z ##### # ##### ###### # #####
1272024-02-13T21:39:18.767Z # # # # # # # # #
1282024-02-13T21:39:18.767Z # # ###### # # # # # #
1292024-02-13T21:39:18.767Z
1302024-02-13T21:39:18.767Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-13T21:39:18.922ZTurn off color for downstairs dump
1322024-02-13T21:39:18.931ZCreating region directory at: /var/tmp/test_repair
1332024-02-13T21:39:18.934ZCreating dsc directory at: /tmp/dsc
1342024-02-13T21:39:20.586ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 1.6451368
1352024-02-13T21:39:21.204ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.6209611
1362024-02-13T21:39:21.373ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.17197196
1372024-02-13T21:39:21.373ZRegion set with 3 regions was created
1382024-02-13T21:39:21.389Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-13T21:39:21.389ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-13T21:39:21.865Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.264990548Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
1412024-02-13T21:39:21.910Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.265710397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
1422024-02-13T21:39:21.911Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.265768356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
1432024-02-13T21:39:21.911Z{"msg":"Crucible ea955154-5f98-40cb-9eb3-3d091db53e99 has session id: ec06ef9d-ac37-4000-bb91-610a8b58466c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.265790038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1442024-02-13T21:39:21.911Z{"msg":"Upstairs opts: Upstairs UUID: ea955154-5f98-40cb-9eb3-3d091db53e99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.265822296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1452024-02-13T21:39:21.911Z{"msg":"Crucible stats registered with UUID: ea955154-5f98-40cb-9eb3-3d091db53e99","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.265855954Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1462024-02-13T21:39:21.911ZCrucible runtime is spawned
1472024-02-13T21:39:21.912Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266272143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
1482024-02-13T21:39:21.912Z{"msg":"ea955154-5f98-40cb-9eb3-3d091db53e99 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266384082Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1492024-02-13T21:39:21.912Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266410543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1502024-02-13T21:39:21.912Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266426027Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1512024-02-13T21:39:21.912Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.26644664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1522024-02-13T21:39:21.912Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266464553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1532024-02-13T21:39:21.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266497371Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1542024-02-13T21:39:21.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266547803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1552024-02-13T21:39:21.913Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266579961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1562024-02-13T21:39:21.913Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266647337Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1572024-02-13T21:39:21.913Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266691701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1582024-02-13T21:39:21.914Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266833589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1592024-02-13T21:39:21.914Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266880582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1602024-02-13T21:39:21.914Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.266908462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1612024-02-13T21:39:21.914Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.269260829Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1622024-02-13T21:39:21.914Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.269296336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1632024-02-13T21:39:21.915Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.26931351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1642024-02-13T21:39:21.915Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.269378286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1652024-02-13T21:39:21.915Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.269437455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1662024-02-13T21:39:21.915Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.269468953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1672024-02-13T21:39:21.915Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.270571303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1682024-02-13T21:39:21.915Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.270721389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1692024-02-13T21:39:21.916Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.270777009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1702024-02-13T21:39:21.916Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271501856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1712024-02-13T21:39:21.916Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271548259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1722024-02-13T21:39:21.916Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271651232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1732024-02-13T21:39:21.916Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271666736Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1742024-02-13T21:39:21.916Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271681641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1752024-02-13T21:39:21.917Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271695446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1762024-02-13T21:39:21.917Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271708651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1772024-02-13T21:39:21.917Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271722336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1782024-02-13T21:39:21.917Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271868463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1792024-02-13T21:39:21.917Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.271896933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1802024-02-13T21:39:21.918Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272039671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1812024-02-13T21:39:21.918Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272074688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1822024-02-13T21:39:21.918Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272092052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1832024-02-13T21:39:21.918Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272118892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1842024-02-13T21:39:21.918Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272133967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1852024-02-13T21:39:21.919Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272159438Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1862024-02-13T21:39:21.919Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272186208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1872024-02-13T21:39:21.919Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272203651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1882024-02-13T21:39:21.919Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272226353Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1892024-02-13T21:39:21.919Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272241957Z"Wait for a query_work_queue command to finish before sending IO
1902024-02-13T21:39:21.919Z,"hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1912024-02-13T21:39:21.919Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272279234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1922024-02-13T21:39:21.920Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272312772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1932024-02-13T21:39:21.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272326537Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1942024-02-13T21:39:21.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272340752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1952024-02-13T21:39:21.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272360485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1962024-02-13T21:39:21.920Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.27237414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1972024-02-13T21:39:21.920Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272393073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1982024-02-13T21:39:21.921Z{"msg":"ea955154-5f98-40cb-9eb3-3d091db53e99 is now active with session: ec06ef9d-ac37-4000-bb91-610a8b58466c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272406378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
1992024-02-13T21:39:21.921Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272418993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2002024-02-13T21:39:21.921Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:20.272437546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
2012024-02-13T21:39:21.921ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-13T21:39:21.921ZFill test
2032024-02-13T21:39:21.921ZRead and Verify all blocks (0..600 range:false)
2042024-02-13T21:39:21.922ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-13T21:39:21.922ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-13T21:39:21.922Z----------------------------------------------------------------
2072024-02-13T21:39:21.922Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-13T21:39:21.922ZStates: Active Active Active
2092024-02-13T21:39:21.922ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-13T21:39:21.923Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-13T21:39:21.923Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-13T21:39:21.923Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-13T21:39:21.923Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-13T21:39:21.923Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-13T21:39:21.923Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-13T21:39:21.923Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-13T21:39:21.923Z New 0 0 0 0
2182024-02-13T21:39:21.923Z Sent 0 0 0 0
2192024-02-13T21:39:21.924Z Done 6 6 6 18
2202024-02-13T21:39:21.924Z Skipped 0 0 0 0
2212024-02-13T21:39:21.924Z Error 0 0 0 0
2222024-02-13T21:39:21.924ZLast Flush: 1006 1006 1006
2232024-02-13T21:39:21.924ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-13T21:39:21.924ZUpstairs last five completed: 13 12 11 10 9
2252024-02-13T21:39:21.924ZCLIENT: Up:0 ds:6 act:3
2262024-02-13T21:39:25.611Z----------------------------------------------------------------
2272024-02-13T21:39:25.612Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-13T21:39:25.612ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-13T21:39:25.612ZUpstairs last five completed: 14 13 12 11 10
2302024-02-13T21:39:25.615ZCLIENT: Up:0 ds:0 act:3
2312024-02-13T21:39:25.615ZCLIENT: All crucible jobs finished, exiting program
2322024-02-13T21:39:25.615Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463421355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141}
2332024-02-13T21:39:25.615Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463555067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2342024-02-13T21:39:25.615Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463632039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2352024-02-13T21:39:25.615Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463709671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2362024-02-13T21:39:25.615Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463780865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2372024-02-13T21:39:25.615Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.463847231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2382024-02-13T21:39:25.615Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463899332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2392024-02-13T21:39:25.615Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.463935989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2402024-02-13T21:39:25.615Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.463962309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"ec06ef9d-ac37-4000-bb91-610a8b58466c"}
2412024-02-13T21:39:25.619Z
2422024-02-13T21:39:25.619ZBegin loop 1
2432024-02-13T21:39:25.619ZDownstairs to restart: 1
2442024-02-13T21:39:25.635Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486438786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
2452024-02-13T21:39:25.635Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486692384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
2462024-02-13T21:39:25.635Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
2472024-02-13T21:39:25.640Z,"v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.48673084Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
2482024-02-13T21:39:25.640Z{"msg":"Crucible d12fdb7b-a02f-454b-8c72-0d39afd4e947 has session id: 82648ec9-6a13-4dc1-a647-9b456a352d24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486763068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2492024-02-13T21:39:25.641Z{"msg":"Upstairs opts: Upstairs UUID: d12fdb7b-a02f-454b-8c72-0d39afd4e947, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486806583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2502024-02-13T21:39:25.641Z{"msg":"Crucible stats registered with UUID: d12fdb7b-a02f-454b-8c72-0d39afd4e947","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486849847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2512024-02-13T21:39:25.641Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486878576Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
2522024-02-13T21:39:25.641Z{"msg":"d12fdb7b-a02f-454b-8c72-0d39afd4e947 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.486982929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2532024-02-13T21:39:25.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487053593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2542024-02-13T21:39:25.641Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487097467Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2552024-02-13T21:39:25.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487137622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2562024-02-13T21:39:25.641Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487219193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2572024-02-13T21:39:25.641Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.48728257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2582024-02-13T21:39:25.641Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487349745Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2592024-02-13T21:39:25.641Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487396409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2602024-02-13T21:39:25.641Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.48744792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2612024-02-13T21:39:25.641Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.48747613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2622024-02-13T21:39:25.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487508368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2632024-02-13T21:39:25.641Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.487538637Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2642024-02-13T21:39:25.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487563628Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2652024-02-13T21:39:25.641Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.487596326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2662024-02-13T21:39:25.641Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:24.487627575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2672024-02-13T21:39:25.641Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487651316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2682024-02-13T21:39:25.641Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487682225Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2692024-02-13T21:39:25.641Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.487738095Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2702024-02-13T21:39:25.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488050621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2712024-02-13T21:39:25.642Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488092696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2722024-02-13T21:39:25.642Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488114218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2732024-02-13T21:39:25.642Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488150945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2742024-02-13T21:39:25.642Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488693128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2752024-02-13T21:39:25.642Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.488730175Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2762024-02-13T21:39:25.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.489195566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2772024-02-13T21:39:25.642Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.489232103Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2782024-02-13T21:39:25.642Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.489247087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2792024-02-13T21:39:25.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.48926707Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2802024-02-13T21:39:25.642Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.489287213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2812024-02-13T21:39:25.642Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:24.489300928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2822024-02-13T21:39:26.638Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.489205057Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2832024-02-13T21:39:26.639Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.489357981Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2842024-02-13T21:39:26.653Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.490256015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2852024-02-13T21:39:26.653Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.49029778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2862024-02-13T21:39:26.654Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.490777616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2872024-02-13T21:39:26.654Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491289331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2882024-02-13T21:39:26.654Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491333405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2892024-02-13T21:39:26.654Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491351678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2902024-02-13T21:39:26.654Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.49137445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2912024-02-13T21:39:26.654Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491389404Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2922024-02-13T21:39:26.654Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491410007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2932024-02-13T21:39:26.654Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491423932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2942024-02-13T21:39:26.654Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491444264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2952024-02-13T21:39:26.654Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.49145825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2962024-02-13T21:39:26.654Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491471985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2972024-02-13T21:39:26.654Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491492587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2982024-02-13T21:39:26.654Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491506092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
2992024-02-13T21:39:26.654Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
3002024-02-13T21:39:26.654Zlevel":30,"time":"2024-02-13T21:39:25.491522936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3012024-02-13T21:39:26.654Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491543029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3022024-02-13T21:39:26.654Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491556804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3032024-02-13T21:39:26.655Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491570759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3042024-02-13T21:39:26.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491583354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3052024-02-13T21:39:26.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491598189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3062024-02-13T21:39:26.655Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491611804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3072024-02-13T21:39:26.655Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491625809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3082024-02-13T21:39:26.655Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491645182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3092024-02-13T21:39:26.655Z{"msg":"d12fdb7b-a02f-454b-8c72-0d39afd4e947 is now active with session: 82648ec9-6a13-4dc1-a647-9b456a352d24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491658347Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3102024-02-13T21:39:26.655Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491670262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143,"session_id":"82648ec9-6a13-4dc1-a647-9b456a352d24"}
3112024-02-13T21:39:26.655Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:25.491690355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
3122024-02-13T21:39:26.655ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3132024-02-13T21:39:26.655ZLoading write count information from file "/tmp/test_repair_verify.data"
3142024-02-13T21:39:26.655ZRun Repair workload
3152024-02-13T21:39:26.655Z01/30 Write block 264 len 20480 data: 2 2 2 2 2
3162024-02-13T21:39:26.662Z02/30 Read block 442 len 40960
3172024-02-13T21:39:26.662Z03/30 Write block 5 len 16384 data: 2 2 2 2
3182024-02-13T21:39:26.662Z04/30 Write block 381 len 28672 data: 2 2 2 2 2 2 2
3192024-02-13T21:39:26.662Z05/30 Read block 115 len 4096
3202024-02-13T21:39:26.662Z06/30 Read block 290 len 24576
3212024-02-13T21:39:26.662Z07/30 Write block 183 len 12288 data: 2 2 2
3222024-02-13T21:39:26.662Z08/30 Read block 481 len 16384
3232024-02-13T21:39:26.662Z09/30 Flush
3242024-02-13T21:39:26.662Z10/30 Write block 125 len 12288 data: 2 2 2
3252024-02-13T21:39:26.662Z11/30 Flush
3262024-02-13T21:39:26.662Z12/30 Write block 230 len 12288 data: 2 2 2
3272024-02-13T21:39:26.663Z13/30 Write block 68 len 20480 data: 2 2 2 2 2
3282024-02-13T21:39:26.663Z14/30 Read block 329 len 28672
3292024-02-13T21:39:26.663Z15/30 Read block 275 len 12288
3302024-02-13T21:39:26.663Z16/30 Read block 329 len 20480
3312024-02-13T21:39:26.663Z17/30 Read block 392 len 32768
3322024-02-13T21:39:26.663Z18/30 Read block 101 len 20480
3332024-02-13T21:39:26.663Z19/30 Read block 405 len 8192
3342024-02-13T21:39:26.666Z20/30 Write block 142 len 16384 data: 2 2 2 2
3352024-02-13T21:39:26.666Z21/30 Write block 183 len 4096 data: 3
3362024-02-13T21:39:26.666Z22/30 Read block 557 len 40960
3372024-02-13T21:39:26.666Z23/30 Read block 358 len 24576
3382024-02-13T21:39:26.666Z24/30 Read block 289 len 4096
3392024-02-13T21:39:26.666Z25/30 Read block 190 len 40960
3402024-02-13T21:39:26.669Z26/30 Write block 320 len 8192 data: 2 2
3412024-02-13T21:39:26.669Z27/30 Write block 204 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3422024-02-13T21:39:26.669Z28/30 Write block 227 len 28672 data: 2 2 2 3 3 3 2
3432024-02-13T21:39:26.669Z29/30 Read block 405 len 28672
3442024-02-13T21:39:26.673Z30/30 Read block 169 len 36864
3452024-02-13T21:39:26.673Z----------------------------------------------------------------
3462024-02-13T21:39:26.673Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:19
3472024-02-13T21:39:26.673ZStates: Active Active Active
3482024-02-13T21:39:26.674ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3492024-02-13T21:39:26.674Z 12 Acked 1011 Write 3 Done Sent Done false
3502024-02-13T21:39:26.674Z 13 Acked 1012 Write 5 Done Sent Done false
3512024-02-13T21:39:26.674Z 14 Acked 1013 Read 7 Done Sent Done false
3522024-02-13T21:39:26.674Z 15 Acked 1014 Read 3 Done Sent Done false
3532024-02-13T21:39:26.674Z 16 Acked 1015 Read 5 Done Sent Done false
3542024-02-13T21:39:26.674Z 17 Acked 1016 Read 8 Done Sent Done false
3552024-02-13T21:39:26.674Z 18 Acked 1017 Read 5 Done Sent Done false
3562024-02-13T21:39:26.674Z 19 Acked 1018 Read 2 Done Sent Done false
3572024-02-13T21:39:26.674Z 20 Acked 1019 Write 4 Done Sent Done false
3582024-02-13T21:39:26.674Z 21 Acked 1020 Write 1 Done Sent Done false
3592024-02-13T21:39:26.674Z 22 Acked 1021 Read 10 Done Sent Done false
3602024-02-13T21:39:26.674Z 23 Acked 1022 Read 6 Done Sent Done false
3612024-02-13T21:39:26.674Z 24 Acked 1023 Read 1 Done Sent Done false
3622024-02-13T21:39:26.674Z 25 Acked 1024 Read 10 Done Sent Done false
3632024-02-13T21:39:26.674Z 26 Acked 1025 Write 2 Done Sent Done false
3642024-02-13T21:39:26.674Z 27 Acked 1026 Write 10 Done Sent Done false
3652024-02-13T21:39:26.674Z 28 Acked 1027 Write 7 Done Sent Done false
3662024-02-13T21:39:26.674Z 29 Acked 1028 Read 7 Done Sent Done false
3672024-02-13T21:39:26.674Z 30 Acked 1029 Read 9 Done Sent Done false
3682024-02-13T21:39:26.674Z STATES DS:0 DS:1 DS:2 TOTAL
3692024-02-13T21:39:26.674Z New 0 0 0 0
3702024-02-13T21:39:26.674Z Sent 0 19 0 19
3712024-02-13T21:39:26.674Z Done 19 0 19 38
3722024-02-13T21:39:26.674Z Skipped 0 0 0 0
3732024-02-13T21:39:26.675Z Error 0 0 0 0
3742024-02-13T21:39:26.675ZLast Flush: 1010 1010 1010
3752024-02-13T21:39:26.675ZDownstairs last five completed: 1010 1009 1008 1007 1006
3762024-02-13T21:39:26.675ZUpstairs last five completed: 30 29 28 27 26
3772024-02-13T21:39:26.675Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:25.52269205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1143}
3782024-02-13T21:39:26.675ZWrote out file "/tmp/test_repair_verify.data"
3792024-02-13T21:39:26.678Z
3802024-02-13T21:39:26.681ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3812024-02-13T21:39:26.695ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3822024-02-13T21:39:26.699Z 0 000-019 2 2 2 2 2 2 F F F
3832024-02-13T21:39:26.699Z 1 020-039 1 1 1 1 1 1 F F F
3842024-02-13T21:39:26.699Z 2 040-059 1 1 1 1 1 1 F F F
3852024-02-13T21:39:26.699Z 3 060-079 1 1 1 1 1 1 T F T <---
3862024-02-13T21:39:26.699Z 4 080-099 1 1 1 1 1 1 F F F
3872024-02-13T21:39:26.699Z 5 100-119 1 1 1 1 1 1 F F F
3882024-02-13T21:39:26.699Z 6 120-139 2 2 2 3 3 3 F F F
3892024-02-13T21:39:26.699Z 7 140-159 1 1 1 1 1 1 T F T <---
3902024-02-13T21:39:26.699Z 8 160-179 1 1 1 1 1 1 F F F
3912024-02-13T21:39:26.699Z 9 180-199 2 2 2 2 2 2 T F T <---
3922024-02-13T21:39:26.700Z 10 200-219 1 1 1 1 1 1 T F T <---
3932024-02-13T21:39:26.700Z 11 220-239 1 1 1 1 1 1 T F T <---
3942024-02-13T21:39:26.700Z 12 240-259 1 1 1 1 1 1 F F F
3952024-02-13T21:39:26.700Z 13 260-279 2 2 2 2 2 2 F F F
3962024-02-13T21:39:26.700Z 14 280-299 1 1 1 1 1 1 F F F
3972024-02-13T21:39:26.700Z 15 300-319 1 1 1 1 1 1 F F F
3982024-02-13T21:39:26.700Z 16 320-339 1 1 1 1 1 1 T F T <---
3992024-02-13T21:39:26.700Z 17 340-359 1 1 1 1 1 1 F F F
4002024-02-13T21:39:26.700Z 18 360-379 1 1 1 1 1 1 F F F
4012024-02-13T21:39:26.700Z 19 380-399 2 2 2 2 2 2 F F F
4022024-02-13T21:39:26.700Z 20 400-419 1 1 1 1 1 1 F F F
4032024-02-13T21:39:26.700Z 21 420-439 1 1 1 1 1 1 F F F
4042024-02-13T21:39:26.700Z 22 440-459 1 1 1 1 1 1 F F F
4052024-02-13T21:39:26.700Z 23 460-479 1 1 1 1 1 1 F F F
4062024-02-13T21:39:26.700Z 24 480-499 1 1 1 1 1 1 F F F
4072024-02-13T21:39:26.700Z 25 500-519 1 1 1 1 1 1 F F F
4082024-02-13T21:39:26.700Z 26 520-539 1 1 1 1 1 1 F F F
4092024-02-13T21:39:26.700Z 27 540-559 1 1 1 1 1 1 F F F
4102024-02-13T21:39:26.700Z 28 560-579 1 1 1 1 1 1 F F F
4112024-02-13T21:39:26.700Z 29 580-599 1 1 1 1 1 1 F F F
4122024-02-13T21:39:26.700ZMax gen: 2, Max flush: 3
4132024-02-13T21:39:26.700ZError: Difference in extent metadata found!
4142024-02-13T21:39:26.700ZA Difference in extent metadata is expected here
4152024-02-13T21:39:26.700ZOn loop 1
4162024-02-13T21:39:26.700Z
4172024-02-13T21:39:26.704ZVerifying data now
4182024-02-13T21:39:31.152ZNo change after verify
4192024-02-13T21:39:31.588ZLoop: 1 Downstairs dump after verify (and repair):
4202024-02-13T21:39:31.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4212024-02-13T21:39:31.588Z 0 000-019 2 2 2 2 2 2 F F F
4222024-02-13T21:39:31.588Z 1 020-039 1 1 1 1 1 1 F F F
4232024-02-13T21:39:31.589Z 2 040-059 1 1 1 1 1 1 F F F
4242024-02-13T21:39:31.589Z 3 060-079 3 3 3 4 4 4 F F F
4252024-02-13T21:39:31.589Z 4 080-099 1 1 1 1 1 1 F F F
4262024-02-13T21:39:31.589Z 5 100-119 1 1 1 1 1 1 F F F
4272024-02-13T21:39:31.589Z 6 120-139 2 2 2 3 3 3 F F F
4282024-02-13T21:39:31.589Z 7 140-159 3 3 3 4 4 4 F F F
4292024-02-13T21:39:31.589Z 8 160-179 1 1 1 1 1 1 F F F
4302024-02-13T21:39:31.589Z 9 180-199 3 3 3 4 4 4 F F F
4312024-02-13T21:39:31.589Z 10 200-219 3 3 3 4 4 4 F F F
4322024-02-13T21:39:31.589Z 11 220-239 3 3 3 4 4 4 F F F
4332024-02-13T21:39:31.590Z 12 240-259 1 1 1 1 1 1 F F F
4342024-02-13T21:39:31.590Z 13 260-279 2 2 2 2 2 2 F F F
4352024-02-13T21:39:31.590Z 14 280-299 1 1 1 1 1 1 F F F
4362024-02-13T21:39:31.590Z 15 300-319 1 1 1 1 1 1 F F F
4372024-02-13T21:39:31.590Z 16 320-339 3 3 3 4 4 4 F F F
4382024-02-13T21:39:31.590Z 17 340-359 1 1 1 1 1 1 F F F
4392024-02-13T21:39:31.590Z 18 360-379 1 1 1 1 1 1 F F F
4402024-02-13T21:39:31.590Z 19 380-399 2 2 2 2 2 2 F F F
4412024-02-13T21:39:31.590Z 20 400-419 1 1 1 1 1 1 F F F
4422024-02-13T21:39:31.590Z 21 420-439 1 1 1 1 1 1 F F F
4432024-02-13T21:39:31.590Z 22 440-459 1 1 1 1 1 1 F F F
4442024-02-13T21:39:31.591Z 23 460-479 1 1 1 1 1 1 F F F
4452024-02-13T21:39:31.591Z 24 480-499 1 1 1 1 1 1 F F F
4462024-02-13T21:39:31.591Z 25 500-519 1 1 1 1 1 1 F F F
4472024-02-13T21:39:31.591Z 26 520-539 1 1 1 1 1 1 F F F
4482024-02-13T21:39:31.591Z 27 540-559 1 1 1 1 1 1 F F F
4492024-02-13T21:39:31.591Z 28 560-579 1 1 1 1 1 1 F F F
4502024-02-13T21:39:31.591Z 29 580-599 1 1 1 1 1 1 F F F
4512024-02-13T21:39:31.591ZMax gen: 3, Max flush: 4
4522024-02-13T21:39:31.591Z
4532024-02-13T21:39:31.592ZBegin loop 2
4542024-02-13T21:39:31.592ZDownstairs to restart: 0
4552024-02-13T21:39:31.637Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023602137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151}
4562024-02-13T21:39:31.637Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023853106Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151}
4572024-02-13T21:39:31.637Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023883645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151Crucible runtime is spawned
4582024-02-13T21:39:31.637Z}
4592024-02-13T21:39:31.637Z{"msg":"Crucible 9edae1e2-a7e8-4893-be85-15ee362ad30e has session id: 8594c87f-8947-4e69-af1c-b1f175b9be18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023912085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4602024-02-13T21:39:31.637Z{"msg":"Upstairs opts: Upstairs UUID: 9edae1e2-a7e8-4893-be85-15ee362ad30e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023955429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4612024-02-13T21:39:31.637Z{"msg":"Crucible stats registered with UUID: 9edae1e2-a7e8-4893-be85-15ee362ad30e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.023991776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4622024-02-13T21:39:31.637Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024016927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151}
4632024-02-13T21:39:31.637Z{"msg":"9edae1e2-a7e8-4893-be85-15ee362ad30e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024148119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4642024-02-13T21:39:31.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024177698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4652024-02-13T21:39:31.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024193483Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4662024-02-13T21:39:31.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024222492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4672024-02-13T21:39:31.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024247413Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4682024-02-13T21:39:31.638Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02428445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4692024-02-13T21:39:31.638Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024313099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4702024-02-13T21:39:31.638Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024341029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4712024-02-13T21:39:31.638Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024377196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4722024-02-13T21:39:31.638Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024406246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4732024-02-13T21:39:31.638Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024441133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4742024-02-13T21:39:31.638Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02447607Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4752024-02-13T21:39:31.638Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.024521344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4762024-02-13T21:39:31.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025157803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4772024-02-13T21:39:31.639Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025199458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4782024-02-13T21:39:31.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025263724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4792024-02-13T21:39:31.639Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025305619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4802024-02-13T21:39:31.639Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025802129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4812024-02-13T21:39:31.639Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.025860258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4822024-02-13T21:39:31.639Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026384428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4832024-02-13T21:39:31.639Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026420145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4842024-02-13T21:39:31.639Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026436059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4852024-02-13T21:39:31.639Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026480903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4862024-02-13T21:39:31.639Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026514971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4872024-02-13T21:39:31.639Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026535873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4882024-02-13T21:39:31.639Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026563513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4892024-02-13T21:39:31.640Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.026583946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4902024-02-13T21:39:31.640Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027078776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4912024-02-13T21:39:31.640Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02745443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4922024-02-13T21:39:31.640Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027486698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4932024-02-13T21:39:31.640Z{"msg":"[0]R flush_numbers[0..12]: [2, 1, 1, 4, 1, 1, 3, 4, 1, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02750989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4942024-02-13T21:39:31.640Z{"msg":"[0]R generation[0..12]: [2, 1, 1, 3, 1, 1, 2, 3, 1, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027540209Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4952024-02-13T21:39:31.640Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027560641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4962024-02-13T21:39:31.640Z{"msg":"[1]R flush_numbers[0..12]: [2, 1, 1, 4, 1, 1, 3, 4, 1, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027589671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4972024-02-13T21:39:31.640Z{"msg":"[1]R generation[0..12]: [2, 1, 1, 3, 1, 1, 2, 3, 1, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027607944Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
4982024-02-13T21:39:31.640ZWait for a query_work_queue command to finish before sending IO
4992024-02-13T21:39:31.640Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027640592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5002024-02-13T21:39:31.640Z{"msg":"[2]R flush_numbers[0..12]: [2, 1, 1, 4, 1, 1, 3, 4, 1, 4, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027672361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5012024-02-13T21:39:31.640Z{"msg":"[2]R generation[0..12]: [2, 1, 1, 3, 1, 1, 2, 3, 1, 3, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027704969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5022024-02-13T21:39:31.640Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027721403Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5032024-02-13T21:39:31.640Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027740776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5042024-02-13T21:39:31.640Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027769096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5052024-02-13T21:39:31.640Z{"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02778349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5062024-02-13T21:39:31.640Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027813779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5072024-02-13T21:39:31.640Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027831923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5082024-02-13T21:39:31.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027859963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5092024-02-13T21:39:31.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027875027Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5102024-02-13T21:39:31.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027905426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5112024-02-13T21:39:31.641Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.02792288Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5122024-02-13T21:39:31.641Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-13T21:39:31.641Z,"time":"2024-02-13T21:39:30.0279509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5142024-02-13T21:39:31.641Z{"msg":"9edae1e2-a7e8-4893-be85-15ee362ad30e is now active with session: 8594c87f-8947-4e69-af1c-b1f175b9be18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027971482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5152024-02-13T21:39:31.641Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.027999202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151,"session_id":"8594c87f-8947-4e69-af1c-b1f175b9be18"}
5162024-02-13T21:39:31.641Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:30.028015326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151}
5172024-02-13T21:39:31.641ZLoading write count information from file "/tmp/test_repair_verify.data"
5182024-02-13T21:39:31.641ZRun Repair workload
5192024-02-13T21:39:31.641Z01/30 Flush
5202024-02-13T21:39:31.641Z02/30 Write block 15 len 32768 data: 2 2 2 2 2 2 2 2
5212024-02-13T21:39:31.641Z03/30 Write block 543 len 16384 data: 2 2 2 2
5222024-02-13T21:39:31.642Z04/30 Flush
5232024-02-13T21:39:31.642Z05/30 Write block 31 len 8192 data: 2 2
5242024-02-13T21:39:31.642Z06/30 Read block 358 len 4096
5252024-02-13T21:39:31.642Z07/30 Flush
5262024-02-13T21:39:31.642Z08/30 Write block 494 len 28672 data: 2 2 2 2 2 2 2
5272024-02-13T21:39:31.642Z09/30 Read block 181 len 32768
5282024-02-13T21:39:31.642Z10/30 Flush
5292024-02-13T21:39:31.642Z11/30 Write block 583 len 20480 data: 2 2 2 2 2
5302024-02-13T21:39:31.642Z12/30 Read block 107 len 20480
5312024-02-13T21:39:31.642Z13/30 Read block 283 len 16384
5322024-02-13T21:39:31.642Z14/30 Read block 511 len 28672
5332024-02-13T21:39:31.642Z15/30 Read block 31 len 12288
5342024-02-13T21:39:31.642Z16/30 Write block 179 len 32768 data: 2 2 2 2 4 3 3 2
5352024-02-13T21:39:31.642Z17/30 Read block 589 len 28672
5362024-02-13T21:39:31.643Z18/30 Read block 92 len 36864
5372024-02-13T21:39:31.643Z19/30 Write block 306 len 28672 data: 2 2 2 2 2 2 2
5382024-02-13T21:39:31.643Z20/30 Read block 534 len 28672
5392024-02-13T21:39:31.643Z21/30 Write block 74 len 12288 data: 2 2 2
5402024-02-13T21:39:31.643Z22/30 Read block 586 len 40960
5412024-02-13T21:39:31.643Z23/30 Write block 284 len 28672 data: 2 2 2 2 2 2 2
5422024-02-13T21:39:31.643Z24/30 Read block 292 len 12288
5432024-02-13T21:39:31.643Z25/30 Read block 214 len 12288
5442024-02-13T21:39:31.643Z26/30 Read block 319 len 4096
5452024-02-13T21:39:31.643Z27/30 Write block 517 len 24576 data: 2 2 2 2 2 2
5462024-02-13T21:39:31.643Z28/30 Write block 96 len 8192 data: 2 2
5472024-02-13T21:39:31.643Z29/30 Read block 2 len 20480
5482024-02-13T21:39:31.643Z30/30 Read block 228 len 4096
5492024-02-13T21:39:31.643Z----------------------------------------------------------------
5502024-02-13T21:39:31.643Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5512024-02-13T21:39:31.644ZStates: Active Active Active
5522024-02-13T21:39:31.644ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5532024-02-13T21:39:31.644Z 1 Acked 1000 Flush 0 Sent Done Done false
5542024-02-13T21:39:31.644Z 2 Acked 1001 Write 8 Sent Done Done false
5552024-02-13T21:39:32.073Z 3 Acked 1002 Write 4 Sent Done Done false
5562024-02-13T21:39:32.073Z 4 Acked 1003 Flush 0 Sent Done Done false
5572024-02-13T21:39:32.073Z 5 Acked 1004 Write 2 Sent Done Done false
5582024-02-13T21:39:32.073Z 6 Acked 1005 Read 1 Sent Done Done false
5592024-02-13T21:39:32.073Z 7 Acked 1006 Flush 0 Sent Done Done false
5602024-02-13T21:39:32.074Z 8 Acked 1007 Write 7 Sent Done Done false
5612024-02-13T21:39:32.074Z 9 Acked 1008 Read 8 Sent Done Done false
5622024-02-13T21:39:32.074Z 10 Acked 1009 Flush 0 Sent Done Done false
5632024-02-13T21:39:32.074Z 11 Acked 1010 Write 5 Sent Done Done false
5642024-02-13T21:39:32.075Z 12 Acked 1011 Read 5 Sent Done Done false
5652024-02-13T21:39:32.075Z 13 Acked 1012 Read 4 Sent Done Done false
5662024-02-13T21:39:32.075Z 14 Acked 1013 Read 7 Sent Done Done false
5672024-02-13T21:39:32.075Z 15 Acked 1014 Read 3 Sent Done Done false
5682024-02-13T21:39:32.075Z 16 Acked 1015 Write 8 Sent Done Done false
5692024-02-13T21:39:32.075Z 17 Acked 1016 Read 7 Sent Done Done false
5702024-02-13T21:39:32.076Z 18 Acked 1017 Read 9 Sent Done Done false
5712024-02-13T21:39:32.076Z 19 Acked 1018 Write 7 Sent Done Done false
5722024-02-13T21:39:32.076Z 20 Acked 1019 Read 7 Sent Done Done false
5732024-02-13T21:39:32.076Z 21 Acked 1020 Write 3 Sent Done Done false
5742024-02-13T21:39:32.076Z 22 Acked 1021 Read 10 Sent Done Done false
5752024-02-13T21:39:32.076Z 23 Acked 1022 Write 7 Sent Done Done false
5762024-02-13T21:39:32.077Z 24 Acked 1023 Read 3 Sent Done Done false
5772024-02-13T21:39:32.077Z 25 Acked 1024 Read 3 Sent Done Done false
5782024-02-13T21:39:32.077Z 26 Acked 1025 Read 1 Sent Done Done false
5792024-02-13T21:39:32.077Z 27 Acked 1026 Write 6 Sent Done Done false
5802024-02-13T21:39:32.077Z 28 Acked 1027 Write 2 Sent Done Done false
5812024-02-13T21:39:32.077Z 29 Acked 1028 Read 5 Sent Done Done false
5822024-02-13T21:39:32.078Z 30 Acked 1029 Read 1 Sent Done Done false
5832024-02-13T21:39:32.078Z STATES DS:0 DS:1 DS:2 TOTAL
5842024-02-13T21:39:32.078Z New 0 0 0 0
5852024-02-13T21:39:32.078Z Sent 30 0 0 30
5862024-02-13T21:39:32.078Z Done 0 30 30 60
5872024-02-13T21:39:32.079Z Skipped 0 0 0 0
5882024-02-13T21:39:32.079Z Error 0 0 0 0
5892024-02-13T21:39:32.079ZLast Flush: 0 1009 1009
5902024-02-13T21:39:32.079ZDownstairs last five completed:
5912024-02-13T21:39:32.079ZUpstairs last five completed: 30 29 28 27 26
5922024-02-13T21:39:32.079Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:30.057937483Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1151}
5932024-02-13T21:39:32.079ZWrote out file "/tmp/test_repair_verify.data"
5942024-02-13T21:39:32.080Z
5952024-02-13T21:39:32.080ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5962024-02-13T21:39:32.080ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5972024-02-13T21:39:32.080Z 0 000-019 2 4 4 2 6 6 F F F <---
5982024-02-13T21:39:32.080Z 1 020-039 1 4 4 1 7 7 F F F <---
5992024-02-13T21:39:32.081Z 2 040-059 1 1 1 1 1 1 F F F
6002024-02-13T21:39:32.081Z 3 060-079 3 3 3 4 4 4 F T T <---
6012024-02-13T21:39:32.081Z 4 080-099 1 1 1 1 1 1 F T T <---
6022024-02-13T21:39:32.081Z 5 100-119 1 1 1 1 1 1 F F F
6032024-02-13T21:39:32.081Z 6 120-139 2 2 2 3 3 3 F F F
6042024-02-13T21:39:32.082Z 7 140-159 3 3 3 4 4 4 F F F
6052024-02-13T21:39:32.082Z 8 160-179 1 1 1 1 1 1 F T T <---
6062024-02-13T21:39:32.082Z 9 180-199 3 3 3 4 4 4 F T T <---
6072024-02-13T21:39:32.082Z 10 200-219 3 3 3 4 4 4 F F F
6082024-02-13T21:39:32.082Z 11 220-239 3 3 3 4 4 4 F F F
6092024-02-13T21:39:32.082Z 12 240-259 1 1 1 1 1 1 F F F
6102024-02-13T21:39:32.082Z 13 260-279 2 2 2 2 2 2 F F F
6112024-02-13T21:39:32.083Z 14 280-299 1 1 1 1 1 1 F T T <---
6122024-02-13T21:39:32.083Z 15 300-319 1 1 1 1 1 1 F T T <---
6132024-02-13T21:39:32.083Z 16 320-339 3 3 3 4 4 4 F F F
6142024-02-13T21:39:32.083Z 17 340-359 1 1 1 1 1 1 F F F
6152024-02-13T21:39:32.083Z 18 360-379 1 1 1 1 1 1 F F F
6162024-02-13T21:39:32.083Z 19 380-399 2 2 2 2 2 2 F F F
6172024-02-13T21:39:32.083Z 20 400-419 1 1 1 1 1 1 F F F
6182024-02-13T21:39:32.083Z 21 420-439 1 1 1 1 1 1 F F F
6192024-02-13T21:39:32.084Z 22 440-459 1 1 1 1 1 1 F F F
6202024-02-13T21:39:32.084Z 23 460-479 1 1 1 1 1 1 F F F
6212024-02-13T21:39:32.084Z 24 480-499 1 4 4 1 8 8 F F F <---
6222024-02-13T21:39:32.084Z 25 500-519 1 4 4 1 8 8 F T T <---
6232024-02-13T21:39:32.084Z 26 520-539 1 1 1 1 1 1 F T T <---
6242024-02-13T21:39:32.084Z 27 540-559 1 4 4 1 6 6 F F F <---
6252024-02-13T21:39:32.084Z 28 560-579 1 1 1 1 1 1 F F F
6262024-02-13T21:39:32.084Z 29 580-599 1 1 1 1 1 1 F T T <---
6272024-02-13T21:39:32.085ZMax gen: 4, Max flush: 8
6282024-02-13T21:39:32.085ZError: Difference in extent metadata found!
6292024-02-13T21:39:32.085ZA Difference in extent metadata is expected here
6302024-02-13T21:39:32.085ZOn loop 2
6312024-02-13T21:39:32.085Z
6322024-02-13T21:39:32.085ZVerifying data now
6332024-02-13T21:39:35.886ZNo change after verify
6342024-02-13T21:39:35.886ZLoop: 2 Downstairs dump after verify (and repair):
6352024-02-13T21:39:35.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6362024-02-13T21:39:35.902Z 0 000-019 4 4 4 6 6 6 F F F
6372024-02-13T21:39:35.902Z 1 020-039 4 4 4 7 7 7 F F F
6382024-02-13T21:39:35.902Z 2 040-059 1 1 1 1 1 1 F F F
6392024-02-13T21:39:35.902Z 3 060-079 5 5 5 9 9 9 F F F
6402024-02-13T21:39:35.902Z 4 080-099 5 5 5 9 9 9 F F F
6412024-02-13T21:39:35.902Z 5 100-119 1 1 1 1 1 1 F F F
6422024-02-13T21:39:35.902Z 6 120-139 2 2 2 3 3 3 F F F
6432024-02-13T21:39:35.903Z 7 140-159 3 3 3 4 4 4 F F F
6442024-02-13T21:39:35.903Z 8 160-179 5 5 5 9 9 9 F F F
6452024-02-13T21:39:35.903Z 9 180-199 5 5 5 9 9 9 F F F
6462024-02-13T21:39:35.903Z 10 200-219 3 3 3 4 4 4 F F F
6472024-02-13T21:39:35.903Z 11 220-239 3 3 3 4 4 4 F F F
6482024-02-13T21:39:35.906Z 12 240-259 1 1 1 1 1 1 F F F
6492024-02-13T21:39:35.906Z 13 260-279 2 2 2 2 2 2 F F F
6502024-02-13T21:39:35.906Z 14 280-299 5 5 5 9 9 9 F F F
6512024-02-13T21:39:35.906Z 15 300-319 5 5 5 9 9 9 F F F
6522024-02-13T21:39:35.906Z 16 320-339 3 3 3 4 4 4 F F F
6532024-02-13T21:39:35.906Z 17 340-359 1 1 1 1 1 1 F F F
6542024-02-13T21:39:35.906Z 18 360-379 1 1 1 1 1 1 F F F
6552024-02-13T21:39:35.907Z 19 380-399 2 2 2 2 2 2 F F F
6562024-02-13T21:39:35.907Z 20 400-419 1 1 1 1 1 1 F F F
6572024-02-13T21:39:35.907Z 21 420-439 1 1 1 1 1 1 F F F
6582024-02-13T21:39:35.907Z 22 440-459 1 1 1 1 1 1 F F F
6592024-02-13T21:39:35.907Z 23 460-479 1 1 1 1 1 1 F F F
6602024-02-13T21:39:35.907Z 24 480-499 4 4 4 8 8 8 F F F
6612024-02-13T21:39:35.907Z 25 500-519 5 5 5 9 9 9 F F F
6622024-02-13T21:39:35.907Z 26 520-539 5 5 5 9 9 9 F F F
6632024-02-13T21:39:35.907Z 27 540-559 4 4 4 6 6 6 F F F
6642024-02-13T21:39:35.907Z 28 560-579 1 1 1 1 1 1 F F F
6652024-02-13T21:39:35.907Z 29 580-599 5 5 5 9 9 9 F F F
6662024-02-13T21:39:35.907ZMax gen: 5, Max flush: 9
6672024-02-13T21:39:35.907Z
6682024-02-13T21:39:35.907ZBegin loop 3
6692024-02-13T21:39:35.907ZDownstairs to restart: 2
6702024-02-13T21:39:35.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.768645669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
6712024-02-13T21:39:35.929Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
6722024-02-13T21:39:35.929Z,"time":"2024-02-13T21:39:34.768923339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
6732024-02-13T21:39:35.930Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.768966373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
6742024-02-13T21:39:35.930Z{"msg":"Crucible 3fb19061-f3c3-4ec5-9995-858409d5b8c8 has session id: 23ad3aa4-737c-491f-90b9-2d93c48cb908","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.768995492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6752024-02-13T21:39:35.930Z{"msg":"Upstairs opts: Upstairs UUID: 3fb19061-f3c3-4ec5-9995-858409d5b8c8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769052302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6762024-02-13T21:39:35.930Z{"msg":"Crucible stats registered with UUID: 3fb19061-f3c3-4ec5-9995-858409d5b8c8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.76908622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6772024-02-13T21:39:35.930Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.76914102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
6782024-02-13T21:39:35.930Z{"msg":"3fb19061-f3c3-4ec5-9995-858409d5b8c8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769192011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6792024-02-13T21:39:35.930Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769227368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6802024-02-13T21:39:35.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769264285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6812024-02-13T21:39:35.930Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769300882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6822024-02-13T21:39:35.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769341497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6832024-02-13T21:39:35.930Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769378494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6842024-02-13T21:39:35.930Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769403285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6852024-02-13T21:39:35.930Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769451457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6862024-02-13T21:39:35.930Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769486254Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6872024-02-13T21:39:35.930Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769531128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6882024-02-13T21:39:35.930Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769555259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6892024-02-13T21:39:35.930Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.76958144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6902024-02-13T21:39:35.930Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.769668808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6912024-02-13T21:39:35.930Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.770128511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6922024-02-13T21:39:35.930Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.770164848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6932024-02-13T21:39:35.930Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.770181212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6942024-02-13T21:39:35.930Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.770195807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6952024-02-13T21:39:35.930Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.77068437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6962024-02-13T21:39:35.931Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.770735561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6972024-02-13T21:39:35.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771247965Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6982024-02-13T21:39:35.931Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771284912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
6992024-02-13T21:39:35.931Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771301016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7002024-02-13T21:39:35.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771314611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7012024-02-13T21:39:35.931Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771336043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7022024-02-13T21:39:35.931Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771350478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7032024-02-13T21:39:35.931Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771370521Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7042024-02-13T21:39:35.931Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771384346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7052024-02-13T21:39:35.931Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.771872149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7062024-02-13T21:39:35.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772315598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7072024-02-13T21:39:35.931Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772352995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7082024-02-13T21:39:35.931Z{"msg":"[0]R flush_numbers[0..12]: [6, 7, 1, 9, 9, 1, 3, 4, 9, 9, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772370558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7092024-02-13T21:39:35.931Z{"msg":"[0]R generation[0..12]: [4, 4, 1, 5, 5, 1, 2, 3, 5, 5, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.7723923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7102024-02-13T21:39:35.931Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772407535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7112024-02-13T21:39:35.931Z{"msg":"[1]R flush_numbers[0..12]: [6, 7, 1, 9, 9, 1, 3, 4, 9, 9, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772422379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7122024-02-13T21:39:35.931Z{"msg":"[1]R generation[0..12]: [4, 4, 1, 5, 5, 1, 2, 3, 5, 5, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772443922Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7132024-02-13T21:39:35.931Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772459276Z","hostname":"Wait for a query_work_queue command to finish before sending IO
7142024-02-13T21:39:35.931Zip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7152024-02-13T21:39:35.931Z{"msg":"[2]R flush_numbers[0..12]: [6, 7, 1, 9, 9, 1, 3, 4, 9, 9, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772487016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7162024-02-13T21:39:35.931Z{"msg":"[2]R generation[0..12]: [4, 4, 1, 5, 5, 1, 2, 3, 5, 5, 3, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772501081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7172024-02-13T21:39:35.931Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772522413Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7182024-02-13T21:39:35.931Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772536128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7192024-02-13T21:39:35.931Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772556721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7202024-02-13T21:39:35.931Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772569936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7212024-02-13T21:39:35.932Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772583431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7222024-02-13T21:39:35.932Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772603414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7232024-02-13T21:39:35.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772615749Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7242024-02-13T21:39:35.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772629144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7252024-02-13T21:39:35.932Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772648847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7262024-02-13T21:39:35.932Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772662642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7272024-02-13T21:39:35.932Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772674918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7282024-02-13T21:39:35.932Z{"msg":"3fb19061-f3c3-4ec5-9995-858409d5b8c8 is now active with session: 23ad3aa4-737c-491f-90b9-2d93c48cb908","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.77269474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7292024-02-13T21:39:35.932Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772706976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159,"session_id":"23ad3aa4-737c-491f-90b9-2d93c48cb908"}
7302024-02-13T21:39:35.932Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:34.772719411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
7312024-02-13T21:39:35.932ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7322024-02-13T21:39:35.932ZLoading write count information from file "/tmp/test_repair_verify.data"
7332024-02-13T21:39:35.932ZRun Repair workload
7342024-02-13T21:39:35.932Z01/30 Flush
7352024-02-13T21:39:35.936Z02/30 Write block 2 len 16384 data: 2 2 2 3
7362024-02-13T21:39:35.936Z03/30 Read block 317 len 32768
7372024-02-13T21:39:35.936Z04/30 Read block 196 len 24576
7382024-02-13T21:39:35.936Z05/30 Write block 508 len 8192 data: 2 2
7392024-02-13T21:39:35.936Z06/30 Write block 373 len 12288 data: 2 2 2
7402024-02-13T21:39:35.936Z07/30 Read block 92 len 16384
7412024-02-13T21:39:35.936Z08/30 Write block 353 len 20480 data: 2 2 2 2 2
7422024-02-13T21:39:35.936Z09/30 Flush
7432024-02-13T21:39:35.936Z10/30 Write block 506 len 36864 data: 2 2 3 3 2 2 2 2 2
7442024-02-13T21:39:35.936Z11/30 Read block 136 len 24576
7452024-02-13T21:39:35.939Z12/30 Read block 355 len 8192
7462024-02-13T21:39:35.939Z13/30 Flush
7472024-02-13T21:39:35.942Z14/30 Write block 475 len 28672 data: 2 2 2 2 2 2 2
7482024-02-13T21:39:35.942Z15/30 Read block 28 len 28672
7492024-02-13T21:39:35.942Z16/30 Read block 2 len 24576
7502024-02-13T21:39:35.946Z17/30 Read block 400 len 36864
7512024-02-13T21:39:35.946Z18/30 Read block 163 len 36864
7522024-02-13T21:39:35.946Z19/30 Write block 568 len 20480 data: 2 2 2 2 2
7532024-02-13T21:39:35.947Z20/30 Write block 87 len 8192 data: 2 2
7542024-02-13T21:39:35.947Z21/30 Read block 495 len 24576
7552024-02-13T21:39:35.950Z22/30 Write block 590 len 36864 data: 2 2 2 2 2 2 2 2 2
7562024-02-13T21:39:35.950Z23/30 Read block 270 len 4096
7572024-02-13T21:39:35.950Z24/30 Read block 138 len 40960
7582024-02-13T21:39:35.950Z25/30 Read block 181 len 4096
7592024-02-13T21:39:35.952Z26/30 Read block 548 len 40960
7602024-02-13T21:39:35.952Z27/30 Read block 90 len 32768
7612024-02-13T21:39:35.952Z28/30 Write block 288 len 40960 data: 3 3 3 2 2 2 2 2 2 2
7622024-02-13T21:39:35.956Z29/30 Write block 198 len 16384 data: 2 2 2 2
7632024-02-13T21:39:35.956Z30/30 Read block 456 len 16384
7642024-02-13T21:39:35.956Z----------------------------------------------------------------
7652024-02-13T21:39:35.956Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:29
7662024-02-13T21:39:35.956ZStates: Active Active Active
7672024-02-13T21:39:35.956ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7682024-02-13T21:39:35.956Z 2 Acked 1001 Write 4 Done Done Done false
7692024-02-13T21:39:35.956Z 3 Acked 1002 Read 8 Done Done Done false
7702024-02-13T21:39:35.957Z 4 Acked 1003 Read 6 Done Done Sent false
7712024-02-13T21:39:35.957Z 5 Acked 1004 Write 2 Done Done Sent false
7722024-02-13T21:39:35.957Z 6 Acked 1005 Write 3 Done Done Sent false
7732024-02-13T21:39:35.957Z 7 Acked 1006 Read 4 Done Done Sent false
7742024-02-13T21:39:35.957Z 8 Acked 1007 Write 5 Done Done Sent false
7752024-02-13T21:39:35.957Z 9 Acked 1008 Flush 0 Done Done Sent false
7762024-02-13T21:39:35.957Z 10 Acked 1009 Write 9 Done Done Sent false
7772024-02-13T21:39:35.957Z 11 Acked 1010 Read 6 Done Done Sent false
7782024-02-13T21:39:35.957Z 12 Acked 1011 Read 2 Done Done Sent false
7792024-02-13T21:39:35.957Z 13 Acked 1012 Flush 0 Done Done Sent false
7802024-02-13T21:39:35.957Z 14 Acked 1013 Write 7 Done Done Sent false
7812024-02-13T21:39:35.957Z 15 Acked 1014 Read 7 Done Done Sent false
7822024-02-13T21:39:35.957Z 16 Acked 1015 Read 6 Done Done Sent false
7832024-02-13T21:39:35.957Z 17 Acked 1016 Read 9 Done Done Sent false
7842024-02-13T21:39:35.957Z 18 Acked 1017 Read 9 Done Done Sent false
7852024-02-13T21:39:35.957Z 19 Acked 1018 Write 5 Done Done Sent false
7862024-02-13T21:39:35.957Z 20 Acked 1019 Write 2 Done Done Sent false
7872024-02-13T21:39:35.957Z 21 Acked 1020 Read 6 Done Done Sent false
7882024-02-13T21:39:35.957Z 22 Acked 1021 Write 9 Done Done Sent false
7892024-02-13T21:39:35.957Z 23 Acked 1022 Read 1 Done Done Sent false
7902024-02-13T21:39:35.957Z 24 Acked 1023 Read 10 Done Done Sent false
7912024-02-13T21:39:35.957Z 25 Acked 1024 Read 1 Done Done Sent false
7922024-02-13T21:39:35.957Z 26 Acked 1025 Read 10 Done Done Sent false
7932024-02-13T21:39:35.957Z 27 Acked 1026 Read 8 Done Done Sent false
7942024-02-13T21:39:35.957Z 28 Acked 1027 Write 10 Done Done Sent false
7952024-02-13T21:39:35.958Z 29 Acked 1028 Write 4 Done Done Sent false
7962024-02-13T21:39:35.958Z 30 Acked 1029 Read 4 Done Sent Sent false
7972024-02-13T21:39:35.958Z STATES DS:0 DS:1 DS:2 TOTAL
7982024-02-13T21:39:35.958Z New 0 0 0 0
7992024-02-13T21:39:35.958Z Sent 0 1 27 28
8002024-02-13T21:39:35.958Z Done 29 28 2 59
8012024-02-13T21:39:35.958Z Skipped 0 0 0 0
8022024-02-13T21:39:35.958Z Error 0 0 0 0
8032024-02-13T21:39:35.958ZLast Flush: 1012 1012 1000
8042024-02-13T21:39:35.958ZDownstairs last five completed: 1000
8052024-02-13T21:39:35.958ZUpstairs last five completed: 30 29 28 27 26
8062024-02-13T21:39:35.958Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:34.802508586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1159}
8072024-02-13T21:39:35.958ZWrote out file "/tmp/test_repair_verify.data"
8082024-02-13T21:39:35.961Z
8092024-02-13T21:39:35.964ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8102024-02-13T21:39:35.979ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8112024-02-13T21:39:35.979Z 0 000-019 6 6 4 11 11 6 F F T <---
8122024-02-13T21:39:35.979Z 1 020-039 4 4 4 7 7 7 F F F
8132024-02-13T21:39:35.979Z 2 040-059 1 1 1 1 1 1 F F F
8142024-02-13T21:39:35.979Z 3 060-079 5 5 5 9 9 9 F F F
8152024-02-13T21:39:35.979Z 4 080-099 5 5 5 9 9 9 T T F <---
8162024-02-13T21:39:35.979Z 5 100-119 1 1 1 1 1 1 F F F
8172024-02-13T21:39:35.979Z 6 120-139 2 2 2 3 3 3 F F F
8182024-02-13T21:39:35.979Z 7 140-159 3 3 3 4 4 4 F F F
8192024-02-13T21:39:35.979Z 8 160-179 5 5 5 9 9 9 F F F
8202024-02-13T21:39:35.979Z 9 180-199 5 5 5 9 9 9 T T F <---
8212024-02-13T21:39:35.979Z 10 200-219 3 3 3 4 4 4 T T F <---
8222024-02-13T21:39:35.979Z 11 220-239 3 3 3 4 4 4 F F F
8232024-02-13T21:39:35.986Z 12 240-259 1 1 1 1 1 1 F F F
8242024-02-13T21:39:35.986Z 13 260-279 2 2 2 2 2 2 F F F
8252024-02-13T21:39:35.986Z 14 280-299 5 5 5 9 9 9 T T F <---
8262024-02-13T21:39:35.986Z 15 300-319 5 5 5 9 9 9 F F F
8272024-02-13T21:39:35.986Z 16 320-339 3 3 3 4 4 4 F F F
8282024-02-13T21:39:35.986Z 17 340-359 6 6 1 11 11 1 F F F <---
8292024-02-13T21:39:35.986Z 18 360-379 6 6 1 11 11 1 F F F <---
8302024-02-13T21:39:35.986Z 19 380-399 2 2 2 2 2 2 F F F
8312024-02-13T21:39:35.986Z 20 400-419 1 1 1 1 1 1 F F F
8322024-02-13T21:39:35.986Z 21 420-439 1 1 1 1 1 1 F F F
8332024-02-13T21:39:35.986Z 22 440-459 1 1 1 1 1 1 F F F
8342024-02-13T21:39:35.986Z 23 460-479 1 1 1 1 1 1 T T F <---
8352024-02-13T21:39:35.986Z 24 480-499 4 4 4 8 8 8 T T F <---
8362024-02-13T21:39:35.986Z 25 500-519 6 6 5 12 12 9 F F F <---
8372024-02-13T21:39:35.986Z 26 520-539 5 5 5 9 9 9 F F F
8382024-02-13T21:39:35.986Z 27 540-559 4 4 4 6 6 6 F F F
8392024-02-13T21:39:35.986Z 28 560-579 1 1 1 1 1 1 T T F <---
8402024-02-13T21:39:35.986Z 29 580-599 5 5 5 9 9 9 T T F <---
8412024-02-13T21:39:35.986ZMax gen: 6, Max flush: 12
8422024-02-13T21:39:35.987ZError: Difference in extent metadata found!
8432024-02-13T21:39:35.987ZA Difference in extent metadata is expected here
8442024-02-13T21:39:35.987ZOn loop 3
8452024-02-13T21:39:35.987Z
8462024-02-13T21:39:35.992ZVerifying data now
8472024-02-13T21:39:40.580ZNo change after verify
8482024-02-13T21:39:40.580ZLoop: 3 Downstairs dump after verify (and repair):
8492024-02-13T21:39:40.594ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8502024-02-13T21:39:40.595Z 0 000-019 6 6 6 11 11 11 F F F
8512024-02-13T21:39:40.595Z 1 020-039 4 4 4 7 7 7 F F F
8522024-02-13T21:39:40.595Z 2 040-059 1 1 1 1 1 1 F F F
8532024-02-13T21:39:40.595Z 3 060-079 5 5 5 9 9 9 F F F
8542024-02-13T21:39:40.595Z 4 080-099 7 7 7 13 13 13 F F F
8552024-02-13T21:39:40.595Z 5 100-119 1 1 1 1 1 1 F F F
8562024-02-13T21:39:40.595Z 6 120-139 2 2 2 3 3 3 F F F
8572024-02-13T21:39:40.595Z 7 140-159 3 3 3 4 4 4 F F F
8582024-02-13T21:39:40.595Z 8 160-179 5 5 5 9 9 9 F F F
8592024-02-13T21:39:40.595Z 9 180-199 7 7 7 13 13 13 F F F
8602024-02-13T21:39:40.595Z 10 200-219 7 7 7 13 13 13 F F F
8612024-02-13T21:39:40.595Z 11 220-239 3 3 3 4 4 4 F F F
8622024-02-13T21:39:40.595Z 12 240-259 1 1 1 1 1 1 F F F
8632024-02-13T21:39:40.595Z 13 260-279 2 2 2 2 2 2 F F F
8642024-02-13T21:39:40.595Z 14 280-299 7 7 7 13 13 13 F F F
8652024-02-13T21:39:40.599Z 15 300-319 5 5 5 9 9 9 F F F
8662024-02-13T21:39:40.599Z 16 320-339 3 3 3 4 4 4 F F F
8672024-02-13T21:39:40.599Z 17 340-359 6 6 6 11 11 11 F F F
8682024-02-13T21:39:40.599Z 18 360-379 6 6 6 11 11 11 F F F
8692024-02-13T21:39:40.599Z 19 380-399 2 2 2 2 2 2 F F F
8702024-02-13T21:39:40.599Z 20 400-419 1 1 1 1 1 1 F F F
8712024-02-13T21:39:40.599Z 21 420-439 1 1 1 1 1 1 F F F
8722024-02-13T21:39:40.599Z 22 440-459 1 1 1 1 1 1 F F F
8732024-02-13T21:39:40.599Z 23 460-479 7 7 7 13 13 13 F F F
8742024-02-13T21:39:40.599Z 24 480-499 7 7 7 13 13 13 F F F
8752024-02-13T21:39:40.599Z 25 500-519 6 6 6 12 12 12 F F F
8762024-02-13T21:39:40.599Z 26 520-539 5 5 5 9 9 9 F F F
8772024-02-13T21:39:40.599Z 27 540-559 4 4 4 6 6 6 F F F
8782024-02-13T21:39:40.599Z 28 560-579 7 7 7 13 13 13 F F F
8792024-02-13T21:39:40.599Z 29 580-599 7 7 7 13 13 13 F F F
8802024-02-13T21:39:40.600ZMax gen: 7, Max flush: 13
8812024-02-13T21:39:40.600Z
8822024-02-13T21:39:40.600ZBegin loop 4
8832024-02-13T21:39:40.600ZDownstairs to restart: 2
8842024-02-13T21:39:40.615Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46056859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
8852024-02-13T21:39:40.620Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.460858625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
8862024-02-13T21:39:40.620Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.460892193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
8872024-02-13T21:39:40.620Z{"msg":"Crucible 7aa798ca-e767-4121-852a-c62ecbb4534b has session id: 15b7fe1a-6a85-49ce-b931-ef9ba819cb9f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.460918943Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8882024-02-13T21:39:40.620Z{"msg":"Crucible runtime is spawned
8892024-02-13T21:39:40.621ZUpstairs opts: Upstairs UUID: 7aa798ca-e767-4121-852a-c62ecbb4534b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.460978681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8902024-02-13T21:39:40.621Z{"msg":"Crucible stats registered with UUID: 7aa798ca-e767-4121-852a-c62ecbb4534b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461019117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8912024-02-13T21:39:40.621Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461044218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
8922024-02-13T21:39:40.621Z{"msg":"7aa798ca-e767-4121-852a-c62ecbb4534b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461103146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8932024-02-13T21:39:40.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461132046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8942024-02-13T21:39:40.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461150399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8952024-02-13T21:39:40.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461187535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8962024-02-13T21:39:40.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461219184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8972024-02-13T21:39:40.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461268436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8982024-02-13T21:39:40.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461303793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
8992024-02-13T21:39:40.621Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461361332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9002024-02-13T21:39:40.621Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461421121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9012024-02-13T21:39:40.621Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461454109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9022024-02-13T21:39:40.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461498533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9032024-02-13T21:39:40.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461554293Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9042024-02-13T21:39:40.621Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.461594408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9052024-02-13T21:39:40.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.462190472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9062024-02-13T21:39:40.621Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.462227998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9072024-02-13T21:39:40.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.462244812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9082024-02-13T21:39:40.621Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.462264545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9092024-02-13T21:39:40.621Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46271996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9102024-02-13T21:39:40.621Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.462867946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9112024-02-13T21:39:40.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463266392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9122024-02-13T21:39:40.621Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463307317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9132024-02-13T21:39:40.622Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463336966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9142024-02-13T21:39:40.622Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463378191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9152024-02-13T21:39:40.622Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.4634072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9162024-02-13T21:39:40.622Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463456203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9172024-02-13T21:39:40.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463498437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9182024-02-13T21:39:40.622Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463537803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9192024-02-13T21:39:40.622Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.463982512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9202024-02-13T21:39:40.627Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464460638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9212024-02-13T21:39:40.627Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464505252Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9222024-02-13T21:39:40.627Z{"msg":"[0]R flush_numbers[0..12]: [11, 7, 1, 9, 13, 1, 3, 4, 9, 13, 13, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464531153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9232024-02-13T21:39:40.627Z{"msg":"[0]R generation[0..12]: [6, 4, 1, 5, 7, 1, 2, 3, 5, 7, 7, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46456513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9242024-02-13T21:39:40.627Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464600228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9252024-02-13T21:39:40.627Z{"msg":"[1]R flush_numbers[0..12]: [11, 7, 1, 9, 13, 1, 3, 4, 9, 13, 13, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464635415Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9262024-02-13T21:39:40.627Z{"msg":"[1]R generation[0..12]: [6, 4, 1, 5, 7, 1, 2, 3, 5, 7, 7, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464670382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
9272024-02-13T21:39:40.627Zpid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9282024-02-13T21:39:40.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46470341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9292024-02-13T21:39:40.627Z{"msg":"[2]R flush_numbers[0..12]: [11, 7, 1, 9, 13, 1, 3, 4, 9, 13, 13, 4]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464740777Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9302024-02-13T21:39:40.627Z{"msg":"[2]R generation[0..12]: [6, 4, 1, 5, 7, 1, 2, 3, 5, 7, 7, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464781572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9312024-02-13T21:39:40.627Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464821647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9322024-02-13T21:39:40.627Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464860273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9332024-02-13T21:39:40.627Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46489663Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9342024-02-13T21:39:40.627Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464933197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9352024-02-13T21:39:40.627Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.464967764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9362024-02-13T21:39:40.627ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9372024-02-13T21:39:40.627Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465028722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9382024-02-13T21:39:40.627Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465045236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9392024-02-13T21:39:40.628Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465065719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9402024-02-13T21:39:40.628Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465079844Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9412024-02-13T21:39:40.628Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465099636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9422024-02-13T21:39:40.628Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465112222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9432024-02-13T21:39:40.628Z{"msg":"7aa798ca-e767-4121-852a-c62ecbb4534b is now active with session: 15b7fe1a-6a85-49ce-b931-ef9ba819cb9f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465125437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9442024-02-13T21:39:40.628Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.46514395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167,"session_id":"15b7fe1a-6a85-49ce-b931-ef9ba819cb9f"}
9452024-02-13T21:39:40.628Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:39.465156886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
9462024-02-13T21:39:40.628ZLoading write count information from file "/tmp/test_repair_verify.data"
9472024-02-13T21:39:40.628ZRun Repair workload
9482024-02-13T21:39:40.628Z01/30 Write block 302 len 16384 data: 2 2 2 2
9492024-02-13T21:39:40.628Z02/30 Write block 220 len 36864 data: 2 2 2 2 2 2 2 3 3
9502024-02-13T21:39:40.628Z03/30 Read block 144 len 16384
9512024-02-13T21:39:40.628Z04/30 Read block 591 len 4096
9522024-02-13T21:39:40.628Z05/30 Write block 498 len 4096 data: 3
9532024-02-13T21:39:40.628Z06/30 Read block 441 len 8192
9542024-02-13T21:39:40.628Z07/30 Flush
9552024-02-13T21:39:40.632Z08/30 Write block 40 len 16384 data: 2 2 2 2
9562024-02-13T21:39:40.632Z09/30 Write block 335 len 20480 data: 2 2 2 2 2
9572024-02-13T21:39:40.632Z10/30 Read block 23 len 20480
9582024-02-13T21:39:40.632Z11/30 Read block 395 len 24576
9592024-02-13T21:39:40.632Z12/30 Write block 551 len 32768 data: 2 2 2 2 2 2 2 2
9602024-02-13T21:39:40.632Z13/30 Read block 239 len 28672
9612024-02-13T21:39:40.636Z14/30 Flush
9622024-02-13T21:39:40.636Z15/30 Write block 121 len 28672 data: 2 2 2 2 3 3 3
9632024-02-13T21:39:40.636Z16/30 Read block 523 len 40960
9642024-02-13T21:39:40.640Z17/30 Write block 494 len 4096 data: 3
9652024-02-13T21:39:40.640Z18/30 Write block 550 len 8192 data: 2 3
9662024-02-13T21:39:40.640Z19/30 Read block 393 len 8192
9672024-02-13T21:39:40.640Z20/30 Write block 340 len 36864 data: 2 2 2 2 2 2 2 2 2
9682024-02-13T21:39:40.643Z21/30 Write block 573 len 36864 data: 2 2 2 2 2 2 2 2 2
9692024-02-13T21:39:40.644Z22/30 Read block 525 len 16384
9702024-02-13T21:39:40.644Z23/30 Flush
9712024-02-13T21:39:40.647Z24/30 Write block 126 len 16384 data: 4 4 2 2
9722024-02-13T21:39:40.647Z25/30 Read block 207 len 4096
9732024-02-13T21:39:40.647Z26/30 Read block 564 len 4096
9742024-02-13T21:39:40.647Z27/30 Write block 533 len 36864 data: 2 2 2 2 2 2 2 2 2
9752024-02-13T21:39:40.650Z28/30 Write block 328 len 4096 data: 2
9762024-02-13T21:39:40.650Z29/30 Write block 16 len 28672 data: 3 3 3 3 3 3 3
9772024-02-13T21:39:40.651Z30/30 Read block 51 len 20480
9782024-02-13T21:39:40.651Z----------------------------------------------------------------
9792024-02-13T21:39:40.651Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9802024-02-13T21:39:40.651ZStates: Active Active Active
9812024-02-13T21:39:40.651ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9822024-02-13T21:39:40.651Z 1 Acked 1000 Write 4 Done Done Sent false
9832024-02-13T21:39:40.651Z 2 Acked 1001 Write 9 Done Done Sent false
9842024-02-13T21:39:40.651Z 3 Acked 1002 Read 4 Done Done Sent false
9852024-02-13T21:39:40.651Z 4 Acked 1003 Read 1 Done Done Sent false
9862024-02-13T21:39:40.651Z 5 Acked 1004 Write 1 Done Done Sent false
9872024-02-13T21:39:40.651Z 6 Acked 1005 Read 2 Done Done Sent false
9882024-02-13T21:39:40.651Z 7 Acked 1006 Flush 0 Done Done Sent false
9892024-02-13T21:39:40.651Z 8 Acked 1007 Write 4 Done Done Sent false
9902024-02-13T21:39:40.651Z 9 Acked 1008 Write 5 Done Done Sent false
9912024-02-13T21:39:40.651Z 10 Acked 1009 Read 5 Done Done Sent false
9922024-02-13T21:39:40.651Z 11 Acked 1010 Read 6 Done Done Sent false
9932024-02-13T21:39:40.651Z 12 Acked 1011 Write 8 Done Done Sent false
9942024-02-13T21:39:40.651Z 13 Acked 1012 Read 7 Done Done Sent false
9952024-02-13T21:39:40.651Z 14 Acked 1013 Flush 0 Done Done Sent false
9962024-02-13T21:39:40.651Z 15 Acked 1014 Write 7 Done Done Sent false
9972024-02-13T21:39:40.651Z 16 Acked 1015 Read 10 Done Done Sent false
9982024-02-13T21:39:40.651Z 17 Acked 1016 Write 1 Done Done Sent false
9992024-02-13T21:39:40.651Z 18 Acked 1017 Write 2 Done Done Sent false
10002024-02-13T21:39:40.651Z 19 Acked 1018 Read 2 Done Done Sent false
10012024-02-13T21:39:40.651Z 20 Acked 1019 Write 9 Done Done Sent false
10022024-02-13T21:39:40.652Z 21 Acked 1020 Write 9 Done Done Sent false
10032024-02-13T21:39:40.652Z 22 Acked 1021 Read 4 Done Done Sent false
10042024-02-13T21:39:40.652Z 23 Acked 1022 Flush 0 Done Done Sent false
10052024-02-13T21:39:40.652Z 24 Acked 1023 Write 4 Done Done Sent false
10062024-02-13T21:39:40.652Z 25 Acked 1024 Read 1 Done Done Sent false
10072024-02-13T21:39:40.652Z 26 Acked 1025 Read 1 Done Done Sent false
10082024-02-13T21:39:40.652Z 27 Acked 1026 Write 9 Done Done Sent false
10092024-02-13T21:39:40.652Z 28 Acked 1027 Write 1 Done Done Sent false
10102024-02-13T21:39:40.652Z 29 Acked 1028 Write 7 Done Done Sent false
10112024-02-13T21:39:40.652Z 30 Acked 1029 Read 5 Done Done Sent false
10122024-02-13T21:39:40.652Z STATES DS:0 DS:1 DS:2 TOTAL
10132024-02-13T21:39:40.652Z New 0 0 0 0
10142024-02-13T21:39:40.652Z Sent 0 0 30 30
10152024-02-13T21:39:40.652Z Done 30 30 0 60
10162024-02-13T21:39:40.652Z Skipped 0 0 0 0
10172024-02-13T21:39:40.652Z Error 0 0 0 0
10182024-02-13T21:39:40.652ZLast Flush: 1022 1022 0
10192024-02-13T21:39:40.652ZDownstairs last five completed:
10202024-02-13T21:39:40.652ZUpstairs last five completed: 30 29 28 27 26
10212024-02-13T21:39:40.652Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:39.495600153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1167}
10222024-02-13T21:39:40.652ZWrote out file "/tmp/test_repair_verify.data"
10232024-02-13T21:39:40.656Z
10242024-02-13T21:39:40.659ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10252024-02-13T21:39:40.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10262024-02-13T21:39:40.673Z 0 000-019 6 6 6 11 11 11 T T F <---
10272024-02-13T21:39:40.673Z 1 020-039 4 4 4 7 7 7 T T F <---
10282024-02-13T21:39:40.673Z 2 040-059 8 8 1 15 15 1 F F F <---
10292024-02-13T21:39:40.673Z 3 060-079 5 5 5 9 9 9 F F F
10302024-02-13T21:39:40.673Z 4 080-099 7 7 7 13 13 13 F F F
10312024-02-13T21:39:40.673Z 5 100-119 1 1 1 1 1 1 F F F
10322024-02-13T21:39:40.673Z 6 120-139 8 8 2 16 16 3 T T F <---
10332024-02-13T21:39:40.673Z 7 140-159 3 3 3 4 4 4 F F F
10342024-02-13T21:39:40.673Z 8 160-179 5 5 5 9 9 9 F F F
10352024-02-13T21:39:40.673Z 9 180-199 7 7 7 13 13 13 F F F
10362024-02-13T21:39:40.673Z 10 200-219 7 7 7 13 13 13 F F F
10372024-02-13T21:39:40.676Z 11 220-239 8 8 3 14 14 4 F F F <---
10382024-02-13T21:39:40.676Z 12 240-259 1 1 1 1 1 1 F F F
10392024-02-13T21:39:40.677Z 13 260-279 2 2 2 2 2 2 F F F
10402024-02-13T21:39:40.677Z 14 280-299 7 7 7 13 13 13 F F F
10412024-02-13T21:39:40.677Z 15 300-319 8 8 5 14 14 9 F F F <---
10422024-02-13T21:39:40.677Z 16 320-339 8 8 3 15 15 4 T T F <---
10432024-02-13T21:39:40.677Z 17 340-359 8 8 6 16 16 11 F F F <---
10442024-02-13T21:39:40.677Z 18 360-379 6 6 6 11 11 11 F F F
10452024-02-13T21:39:40.677Z 19 380-399 2 2 2 2 2 2 F F F
10462024-02-13T21:39:40.677Z 20 400-419 1 1 1 1 1 1 F F F
10472024-02-13T21:39:40.677Z 21 420-439 1 1 1 1 1 1 F F F
10482024-02-13T21:39:40.677Z 22 440-459 1 1 1 1 1 1 F F F
10492024-02-13T21:39:40.677Z 23 460-479 7 7 7 13 13 13 F F F
10502024-02-13T21:39:40.677Z 24 480-499 8 8 7 16 16 13 F F F <---
10512024-02-13T21:39:40.677Z 25 500-519 6 6 6 12 12 12 F F F
10522024-02-13T21:39:40.677Z 26 520-539 5 5 5 9 9 9 T T F <---
10532024-02-13T21:39:40.677Z 27 540-559 8 8 4 16 16 6 T T F <---
10542024-02-13T21:39:40.677Z 28 560-579 8 8 7 16 16 13 F F F <---
10552024-02-13T21:39:40.677Z 29 580-599 8 8 7 16 16 13 F F F <---
10562024-02-13T21:39:40.677ZMax gen: 8, Max flush: 16
10572024-02-13T21:39:40.677ZError: Difference in extent metadata found!
10582024-02-13T21:39:40.677ZA Difference in extent metadata is expected here
10592024-02-13T21:39:40.677ZOn loop 4
10602024-02-13T21:39:40.677Z
10612024-02-13T21:39:40.681ZVerifying data now
10622024-02-13T21:39:45.305ZNo change after verify
10632024-02-13T21:39:45.309ZLoop: 4 Downstairs dump after verify (and repair):
10642024-02-13T21:39:45.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10652024-02-13T21:39:45.320Z 0 000-019 9 9 9 17 17 17 F F F
10662024-02-13T21:39:45.320Z 1 020-039 9 9 9 17 17 17 F F F
10672024-02-13T21:39:45.320Z 2 040-059 8 8 8 15 15 15 F F F
10682024-02-13T21:39:45.320Z 3 060-079 5 5 5 9 9 9 F F F
10692024-02-13T21:39:45.321Z 4 080-099 7 7 7 13 13 13 F F F
10702024-02-13T21:39:45.321Z 5 100-119 1 1 1 1 1 1 F F F
10712024-02-13T21:39:45.321Z 6 120-139 9 9 9 17 17 17 F F F
10722024-02-13T21:39:45.321Z 7 140-159 3 3 3 4 4 4 F F F
10732024-02-13T21:39:45.321Z 8 160-179 5 5 5 9 9 9 F F F
10742024-02-13T21:39:45.325Z 9 180-199 7 7 7 13 13 13 F F F
10752024-02-13T21:39:45.325Z 10 200-219 7 7 7 13 13 13 F F F
10762024-02-13T21:39:45.326Z 11 220-239 8 8 8 14 14 14 F F F
10772024-02-13T21:39:45.326Z 12 240-259 1 1 1 1 1 1 F F F
10782024-02-13T21:39:45.326Z 13 260-279 2 2 2 2 2 2 F F F
10792024-02-13T21:39:45.326Z 14 280-299 7 7 7 13 13 13 F F F
10802024-02-13T21:39:45.326Z 15 300-319 8 8 8 14 14 14 F F F
10812024-02-13T21:39:45.326Z 16 320-339 9 9 9 17 17 17 F F F
10822024-02-13T21:39:45.326Z 17 340-359 8 8 8 16 16 16 F F F
10832024-02-13T21:39:45.326Z 18 360-379 6 6 6 11 11 11 F F F
10842024-02-13T21:39:45.327Z 19 380-399 2 2 2 2 2 2 F F F
10852024-02-13T21:39:45.327Z 20 400-419 1 1 1 1 1 1 F F F
10862024-02-13T21:39:45.327Z 21 420-439 1 1 1 1 1 1 F F F
10872024-02-13T21:39:45.327Z 22 440-459 1 1 1 1 1 1 F F F
10882024-02-13T21:39:45.327Z 23 460-479 7 7 7 13 13 13 F F F
10892024-02-13T21:39:45.327Z 24 480-499 8 8 8 16 16 16 F F F
10902024-02-13T21:39:45.328Z 25 500-519 6 6 6 12 12 12 F F F
10912024-02-13T21:39:45.328Z 26 520-539 9 9 9 17 17 17 F F F
10922024-02-13T21:39:45.328Z 27 540-559 9 9 9 17 17 17 F F F
10932024-02-13T21:39:45.328Z 28 560-579 8 8 8 16 16 16 F F F
10942024-02-13T21:39:45.328Z 29 580-599 8 8 8 16 16 16 F F F
10952024-02-13T21:39:45.328ZMax gen: 9, Max flush: 17
10962024-02-13T21:39:45.328Z
10972024-02-13T21:39:45.328ZBegin loop 5
10982024-02-13T21:39:45.329ZDownstairs to restart: 1
10992024-02-13T21:39:45.341Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184577122Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
11002024-02-13T21:39:45.348Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184847004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
11012024-02-13T21:39:45.348ZCrucible runtime is spawned
11022024-02-13T21:39:45.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184889918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
11032024-02-13T21:39:45.349Z{"msg":"Crucible 6e2a3064-caec-4553-b98e-fff93bf64473 has session id: a6fbfc74-da17-4ff9-8dc5-b3ac74a68618","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184917908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11042024-02-13T21:39:45.349Z{"msg":"Upstairs opts: Upstairs UUID: 6e2a3064-caec-4553-b98e-fff93bf64473, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184970739Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11052024-02-13T21:39:45.349Z{"msg":"Crucible stats registered with UUID: 6e2a3064-caec-4553-b98e-fff93bf64473","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.184998809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11062024-02-13T21:39:45.349Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185013304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
11072024-02-13T21:39:45.349Z{"msg":"6e2a3064-caec-4553-b98e-fff93bf64473 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185096603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11082024-02-13T21:39:45.349Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185123593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11092024-02-13T21:39:45.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185138568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11102024-02-13T21:39:45.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185166108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11112024-02-13T21:39:45.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185180933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11122024-02-13T21:39:45.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185197977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11132024-02-13T21:39:45.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185223557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11142024-02-13T21:39:45.350Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185237582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11152024-02-13T21:39:45.350Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185268591Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11162024-02-13T21:39:45.351Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185284635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11172024-02-13T21:39:45.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185475186Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11182024-02-13T21:39:45.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185646004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11192024-02-13T21:39:45.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.185687079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11202024-02-13T21:39:45.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186295108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11212024-02-13T21:39:45.351Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186361244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11222024-02-13T21:39:45.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186404339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11232024-02-13T21:39:45.351Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186431159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11242024-02-13T21:39:45.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.18687135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11252024-02-13T21:39:45.351Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186928009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11262024-02-13T21:39:45.352Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.186959128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11272024-02-13T21:39:45.352Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.18698111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11282024-02-13T21:39:45.352Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187549673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11292024-02-13T21:39:45.352Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187589029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11302024-02-13T21:39:45.352Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.18761556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11312024-02-13T21:39:45.352Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187653246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11322024-02-13T21:39:45.353Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187689853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11332024-02-13T21:39:45.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187716013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11342024-02-13T21:39:45.353Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.187739405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11352024-02-13T21:39:45.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188078912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11362024-02-13T21:39:45.353Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188114379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11372024-02-13T21:39:45.353Z{"msg":"[0]R flush_numbers[0..12]: [17, 17, 15, 9, 13, 1, 17, 4, 9, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188130913Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11382024-02-13T21:39:45.353Z{"msg":"[0]R generation[0..12]: [9, 9, 8, 5, 7, 1, 9, 3, 5, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188146167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11392024-02-13T21:39:45.354Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188161012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11402024-02-13T21:39:45.354Z{"msg":"[1]R flush_numbers[0..12]: [17, 17, 15, 9, 13, 1, 17, 4, 9, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188175397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11412024-02-13T21:39:45.354Z{"msg":"[1]R generation[0..12]: [9, 9, 8, 5, 7, 1, 9, 3, 5, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188196289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11422024-02-13T21:39:45.354Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188211004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11432024-02-13T21:39:45.354Z{"msg":"[2]R flush_numbers[0..12]: [17, 17, 15, 9, 13, 1, 17, 4, 9, 13, 13, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188236384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11442024-02-13T21:39:45.354Z{"msg":"[2]R generation[0..12]: [9, 9, 8, 5, 7, 1, 9, 3, 5, 7, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188268573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11452024-02-13T21:39:45.354Z{"msg":"Wait for a query_work_queue command to finish before sending IO
11462024-02-13T21:39:45.354Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188286166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11472024-02-13T21:39:45.355Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188309588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11482024-02-13T21:39:45.355Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188323123Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11492024-02-13T21:39:45.355Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188341196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11502024-02-13T21:39:45.355Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188373694Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11512024-02-13T21:39:45.355Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188401025Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11522024-02-13T21:39:45.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188438011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11532024-02-13T21:39:45.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188465571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11542024-02-13T21:39:45.355Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188501418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11552024-02-13T21:39:45.355Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188535096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11562024-02-13T21:39:45.356Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188550281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11572024-02-13T21:39:45.356Z{"msg":"6e2a3064-caec-4553-b98e-fff93bf64473 is now active with session: a6fbfc74-da17-4ff9-8dc5-b3ac74a68618","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188577361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11582024-02-13T21:39:45.356Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188611808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175,"session_id":"a6fbfc74-da17-4ff9-8dc5-b3ac74a68618"}
11592024-02-13T21:39:45.356Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:44.188637849Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
11602024-02-13T21:39:45.356ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11612024-02-13T21:39:45.360ZLoading write count information from file "/tmp/test_repair_verify.data"
11622024-02-13T21:39:45.360ZRun Repair workload
11632024-02-13T21:39:45.360Z01/30 Write block 451 len 24576 data: 2 2 2 2 2 2
11642024-02-13T21:39:45.361Z02/30 Write block 213 len 28672 data: 3 2 2 2 2 2 2
11652024-02-13T21:39:45.361Z03/30 Read block 214 len 16384
11662024-02-13T21:39:45.361Z04/30 Write block 434 len 8192 data: 2 2
11672024-02-13T21:39:45.361Z05/30 Read block 585 len 28672
11682024-02-13T21:39:45.361Z06/30 Read block 563 len 36864
11692024-02-13T21:39:45.361Z07/30 Flush
11702024-02-13T21:39:45.362Z08/30 Write block 356 len 32768 data: 3 3 2 2 2 2 2 2
11712024-02-13T21:39:45.362Z09/30 Read block 123 len 8192
11722024-02-13T21:39:45.362Z10/30 Read block 199 len 8192
11732024-02-13T21:39:45.362Z11/30 Write block 238 len 40960 data: 2 2 2 2 2 2 2 2 2 2
11742024-02-13T21:39:45.362Z12/30 Read block 486 len 32768
11752024-02-13T21:39:45.362Z13/30 Read block 383 len 4096
11762024-02-13T21:39:45.362Z14/30 Read block 430 len 32768
11772024-02-13T21:39:45.363Z15/30 Read block 77 len 36864
11782024-02-13T21:39:45.367Z16/30 Read block 298 len 4096
11792024-02-13T21:39:45.367Z17/30 Read block 131 len 36864
11802024-02-13T21:39:45.367Z18/30 Read block 72 len 40960
11812024-02-13T21:39:45.367Z19/30 Write block 31 len 36864 data: 3 3 2 2 2 2 2 2 2
11822024-02-13T21:39:45.367Z20/30 Read block 537 len 16384
11832024-02-13T21:39:45.367Z21/30 Read block 4 len 8192
11842024-02-13T21:39:45.368Z22/30 Read block 400 len 36864
11852024-02-13T21:39:45.372Z23/30 Read block 97 len 40960
11862024-02-13T21:39:45.372Z24/30 Read block 334 len 32768
11872024-02-13T21:39:45.372Z25/30 Write block 216 len 16384 data: 3 3 3 3
11882024-02-13T21:39:45.372Z26/30 Read block 20 len 40960
11892024-02-13T21:39:45.372Z27/30 Read block 181 len 40960
11902024-02-13T21:39:45.377Z28/30 Write block 305 len 8192 data: 3 3
11912024-02-13T21:39:45.377Z29/30 Read block 535 len 4096
11922024-02-13T21:39:45.377Z30/30 Write block 407 len 8192 data: 2 2
11932024-02-13T21:39:45.378Z----------------------------------------------------------------
11942024-02-13T21:39:45.378Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11952024-02-13T21:39:45.378ZStates: Active Active Active
11962024-02-13T21:39:45.379ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11972024-02-13T21:39:45.379Z 1 Acked 1000 Write 6 Done Done Done false
11982024-02-13T21:39:45.379Z 2 Acked 1001 Write 7 Done Sent Done false
11992024-02-13T21:39:45.379Z 3 Acked 1002 Read 4 Done Sent Done false
12002024-02-13T21:39:45.379Z 4 Acked 1003 Write 2 Done Sent Done false
12012024-02-13T21:39:45.379Z 5 Acked 1004 Read 7 Done Sent Done false
12022024-02-13T21:39:45.379Z 6 Acked 1005 Read 9 Done Sent Done false
12032024-02-13T21:39:45.379Z 7 Acked 1006 Flush 0 Done Sent Done false
12042024-02-13T21:39:45.379Z 8 Acked 1007 Write 8 Done Sent Done false
12052024-02-13T21:39:45.379Z 9 Acked 1008 Read 2 Done Sent Done false
12062024-02-13T21:39:45.380Z 10 Acked 1009 Read 2 Done Sent Done false
12072024-02-13T21:39:45.380Z 11 Acked 1010 Write 10 Done Sent Done false
12082024-02-13T21:39:45.380Z 12 Acked 1011 Read 8 Done Sent Done false
12092024-02-13T21:39:45.380Z 13 Acked 1012 Read 1 Done Sent Done false
12102024-02-13T21:39:45.380Z 14 Acked 1013 Read 8 Done Sent Done false
12112024-02-13T21:39:45.380Z 15 Acked 1014 Read 9 Done Sent Done false
12122024-02-13T21:39:45.381Z 16 Acked 1015 Read 1 Done Sent Done false
12132024-02-13T21:39:45.381Z 17 Acked 1016 Read 9 Done Sent Done false
12142024-02-13T21:39:45.381Z 18 Acked 1017 Read 10 Done Sent Done false
12152024-02-13T21:39:45.381Z 19 Acked 1018 Write 9 Done Sent Done false
12162024-02-13T21:39:45.381Z 20 Acked 1019 Read 4 Done Sent Done false
12172024-02-13T21:39:45.382Z 21 Acked 1020 Read 2 Done Sent Done false
12182024-02-13T21:39:45.382Z 22 Acked 1021 Read 9 Done Sent Done false
12192024-02-13T21:39:45.382Z 23 Acked 1022 Read 10 Done Sent Done false
12202024-02-13T21:39:45.382Z 24 Acked 1023 Read 8 Done Sent Done false
12212024-02-13T21:39:45.382Z 25 Acked 1024 Write 4 Done Sent Done false
12222024-02-13T21:39:45.383Z 26 Acked 1025 Read 10 Done Sent Done false
12232024-02-13T21:39:45.383Z 27 Acked 1026 Read 10 Done Sent Done false
12242024-02-13T21:39:45.383Z 28 Acked 1027 Write 2 Done Sent Done false
12252024-02-13T21:39:45.383Z 29 Acked 1028 Read 1 Done Sent Done false
12262024-02-13T21:39:45.383Z 30 Acked 1029 Write 2 Sent Sent Sent false
12272024-02-13T21:39:45.384Z STATES DS:0 DS:1 DS:2 TOTAL
12282024-02-13T21:39:45.384Z New 0 0 0 0
12292024-02-13T21:39:45.384Z Sent 1 29 1 31
12302024-02-13T21:39:45.384Z Done 29 1 29 59
12312024-02-13T21:39:45.384Z Skipped 0 0 0 0
12322024-02-13T21:39:45.384Z Error 0 0 0 0
12332024-02-13T21:39:45.385ZLast Flush: 1006 0 1006
12342024-02-13T21:39:45.385ZDownstairs last five completed:
12352024-02-13T21:39:45.385ZUpstairs last five completed: 30 29 28 27 26
12362024-02-13T21:39:45.385Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:44.217907821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1175}
12372024-02-13T21:39:45.385ZWrote out file "/tmp/test_repair_verify.data"
12382024-02-13T21:39:45.396Z
12392024-02-13T21:39:45.396ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12402024-02-13T21:39:45.411ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12412024-02-13T21:39:45.412Z 0 000-019 9 9 9 17 17 17 F F F
12422024-02-13T21:39:45.412Z 1 020-039 9 9 9 17 17 17 T F T <---
12432024-02-13T21:39:45.412Z 2 040-059 8 8 8 15 15 15 F F F
12442024-02-13T21:39:45.412Z 3 060-079 5 5 5 9 9 9 F F F
12452024-02-13T21:39:45.412Z 4 080-099 7 7 7 13 13 13 F F F
12462024-02-13T21:39:45.412Z 5 100-119 1 1 1 1 1 1 F F F
12472024-02-13T21:39:45.412Z 6 120-139 9 9 9 17 17 17 F F F
12482024-02-13T21:39:45.412Z 7 140-159 3 3 3 4 4 4 F F F
12492024-02-13T21:39:45.412Z 8 160-179 5 5 5 9 9 9 F F F
12502024-02-13T21:39:45.412Z 9 180-199 7 7 7 13 13 13 F F F
12512024-02-13T21:39:45.412Z 10 200-219 10 7 10 18 13 18 T F T <---
12522024-02-13T21:39:45.412Z 11 220-239 8 8 8 14 14 14 T F T <---
12532024-02-13T21:39:45.412Z 12 240-259 1 1 1 1 1 1 T F T <---
12542024-02-13T21:39:45.413Z 13 260-279 2 2 2 2 2 2 F F F
12552024-02-13T21:39:45.413Z 14 280-299 7 7 7 13 13 13 F F F
12562024-02-13T21:39:45.413Z 15 300-319 8 8 8 14 14 14 T F T <---
12572024-02-13T21:39:45.413Z 16 320-339 9 9 9 17 17 17 F F F
12582024-02-13T21:39:45.413Z 17 340-359 8 8 8 16 16 16 T F T <---
12592024-02-13T21:39:45.413Z 18 360-379 6 6 6 11 11 11 T F T <---
12602024-02-13T21:39:45.413Z 19 380-399 2 2 2 2 2 2 F F F
12612024-02-13T21:39:45.413Z 20 400-419 1 1 1 1 1 1 T F T <---
12622024-02-13T21:39:45.413Z 21 420-439 10 1 10 18 1 18 F F F <---
12632024-02-13T21:39:45.413Z 22 440-459 10 1 10 18 1 18 F T F <---
12642024-02-13T21:39:45.413Z 23 460-479 7 7 7 13 13 13 F F F
12652024-02-13T21:39:45.413Z 24 480-499 8 8 8 16 16 16 F F F
12662024-02-13T21:39:45.413Z 25 500-519 6 6 6 12 12 12 F F F
12672024-02-13T21:39:45.413Z 26 520-539 9 9 9 17 17 17 F F F
12682024-02-13T21:39:45.413Z 27 540-559 9 9 9 17 17 17 F F F
12692024-02-13T21:39:45.413Z 28 560-579 8 8 8 16 16 16 F F F
12702024-02-13T21:39:45.413Z 29 580-599 8 8 8 16 16 16 F F F
12712024-02-13T21:39:45.414ZMax gen: 10, Max flush: 18
12722024-02-13T21:39:45.414ZError: Difference in extent metadata found!
12732024-02-13T21:39:45.414ZA Difference in extent metadata is expected here
12742024-02-13T21:39:45.414ZOn loop 5
12752024-02-13T21:39:45.414Z
12762024-02-13T21:39:45.414ZVerifying data now
12772024-02-13T21:39:53.799ZNo change after verify
12782024-02-13T21:39:53.799ZLoop: 5 Downstairs dump after verify (and repair):
12792024-02-13T21:39:53.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12802024-02-13T21:39:53.799Z 0 000-019 9 9 9 17 17 17 F F F
12812024-02-13T21:39:53.800Z 1 020-039 11 11 11 19 19 19 F F F
12822024-02-13T21:39:53.800Z 2 040-059 8 8 8 15 15 15 F F F
12832024-02-13T21:39:53.800Z 3 060-079 5 5 5 9 9 9 F F F
12842024-02-13T21:39:53.800Z 4 080-099 7 7 7 13 13 13 F F F
12852024-02-13T21:39:53.800Z 5 100-119 1 1 1 1 1 1 F F F
12862024-02-13T21:39:53.800Z 6 120-139 9 9 9 17 17 17 F F F
12872024-02-13T21:39:53.801Z 7 140-159 3 3 3 4 4 4 F F F
12882024-02-13T21:39:53.801Z 8 160-179 5 5 5 9 9 9 F F F
12892024-02-13T21:39:53.801Z 9 180-199 7 7 7 13 13 13 F F F
12902024-02-13T21:39:53.801Z 10 200-219 11 11 11 19 19 19 F F F
12912024-02-13T21:39:53.801Z 11 220-239 11 11 11 19 19 19 F F F
12922024-02-13T21:39:53.801Z 12 240-259 11 11 11 19 19 19 F F F
12932024-02-13T21:39:53.801Z 13 260-279 2 2 2 2 2 2 F F F
12942024-02-13T21:39:53.801Z 14 280-299 7 7 7 13 13 13 F F F
12952024-02-13T21:39:53.801Z 15 300-319 11 11 11 19 19 19 F F F
12962024-02-13T21:39:53.802Z 16 320-339 9 9 9 17 17 17 F F F
12972024-02-13T21:39:53.802Z 17 340-359 11 11 11 19 19 19 F F F
12982024-02-13T21:39:53.802Z 18 360-379 11 11 11 19 19 19 F F F
12992024-02-13T21:39:53.802Z 19 380-399 2 2 2 2 2 2 F F F
13002024-02-13T21:39:53.802Z 20 400-419 11 11 11 19 19 19 F F F
13012024-02-13T21:39:53.802Z 21 420-439 10 10 10 18 18 18 F F F
13022024-02-13T21:39:53.802Z 22 440-459 10 10 10 18 18 18 F F F
13032024-02-13T21:39:53.802Z 23 460-479 7 7 7 13 13 13 F F F
13042024-02-13T21:39:53.823Z 24 480-499 8 8 8 16 16 16 F F F
13052024-02-13T21:39:53.824Z 25 500-519 6 6 6 12 12 12 F F F
13062024-02-13T21:39:53.824Z 26 520-539 9 9 9 17 17 17 F F F
13072024-02-13T21:39:53.824Z 27 540-559 9 9 9 17 17 17 F F F
13082024-02-13T21:39:53.824Z 28 560-579 8 8 8 16 16 16 F F F
13092024-02-13T21:39:53.824Z 29 580-599 8 8 8 16 16 16 F F F
13102024-02-13T21:39:53.825ZMax gen: 11, Max flush: 19
13112024-02-13T21:39:53.825Z
13122024-02-13T21:39:53.825ZBegin loop 6
13132024-02-13T21:39:53.825ZDownstairs to restart: 2
13142024-02-13T21:39:53.825Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786390667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
13152024-02-13T21:39:53.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786615476Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
13162024-02-13T21:39:53.826Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786634349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
13172024-02-13T21:39:53.827Z{"msg":"Crucible 9628627b-f4d5-42a8-99c8-fd9856e04289 has session id: a5d22715-6630-412c-b0d2-606a4f238b7c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786647834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13182024-02-13T21:39:53.827Z{"msg":"Upstairs opts: Upstairs UUID: 9628627b-f4d5-42a8-99c8-fd9856e04289, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786669366Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13192024-02-13T21:39:53.827Z{"msg":"Crucible stats registered with UUID: 9628627b-f4d5-42a8-99c8-fd9856e04289","v":0,"name":"crucible","level":30Crucible runtime is spawned
13202024-02-13T21:39:53.827Z,"time":"2024-02-13T21:39:48.786694697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13212024-02-13T21:39:53.827Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786809705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
13222024-02-13T21:39:53.828Z{"msg":"9628627b-f4d5-42a8-99c8-fd9856e04289 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786899673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13232024-02-13T21:39:53.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.786981293Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13242024-02-13T21:39:53.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787029026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13252024-02-13T21:39:53.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787057885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13262024-02-13T21:39:53.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787093312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13272024-02-13T21:39:53.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.78712723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13282024-02-13T21:39:53.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.78715492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13292024-02-13T21:39:53.829Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787189208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13302024-02-13T21:39:53.830Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787227304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13312024-02-13T21:39:53.830Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.78726428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13322024-02-13T21:39:53.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787299597Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13332024-02-13T21:39:53.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787440716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13342024-02-13T21:39:53.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787467867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13352024-02-13T21:39:53.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787940035Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13362024-02-13T21:39:53.831Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787973963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13372024-02-13T21:39:53.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.787988967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13382024-02-13T21:39:53.832Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.788019076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13392024-02-13T21:39:53.832Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.788594908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13402024-02-13T21:39:53.832Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.788639611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13412024-02-13T21:39:53.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.788923179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13422024-02-13T21:39:53.833Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.788970591Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13432024-02-13T21:39:53.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.78925035Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13442024-02-13T21:39:53.833Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789291445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13452024-02-13T21:39:53.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789307499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13462024-02-13T21:39:53.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789327842Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13472024-02-13T21:39:53.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789348414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13482024-02-13T21:39:53.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789368477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13492024-02-13T21:39:53.834Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789478667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13502024-02-13T21:39:53.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789882371Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13512024-02-13T21:39:53.835Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789935261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13522024-02-13T21:39:53.835Z{"msg":"[0]R flush_numbers[0..12]: [17, 19, 15, 9, 13, 1, 17, 4, 9, 13, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789970939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13532024-02-13T21:39:53.835Z{"msg":"[0]R generation[0..12]: [9, 11, 8, 5, 7, 1, 9, 3, 5, 7, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.789993211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13542024-02-13T21:39:53.835Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13552024-02-13T21:39:53.835Z,"time":"2024-02-13T21:39:48.790028568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13562024-02-13T21:39:53.836Z{"msg":"[1]R flush_numbers[0..12]: [17, 19, 15, 9, 13, 1, 17, 4, 9, 13, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790047141Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13572024-02-13T21:39:53.836Z{"msg":"[1]R generation[0..12]: [9, 11, 8, 5, 7, 1, 9, 3, 5, 7, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790067613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13582024-02-13T21:39:53.836Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790094974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13592024-02-13T21:39:53.836Z{"msg":"[2]R flush_numbers[0..12]: [17, 19, 15, 9, 13, 1, 17, 4, 9, 13, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790109858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13602024-02-13T21:39:53.836Z{"msg":"[2]R generation[0..12]: [9, 11, 8, 5, 7, 1, 9, 3, 5, 7, 11, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790124063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13612024-02-13T21:39:53.836Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790138768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13622024-02-13T21:39:53.837Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.79015886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13632024-02-13T21:39:53.837Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790194947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13642024-02-13T21:39:53.837Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.79021489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13652024-02-13T21:39:53.837Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790240491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13662024-02-13T21:39:53.837Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790268211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13672024-02-13T21:39:53.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790295741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13682024-02-13T21:39:53.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790328699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13692024-02-13T21:39:53.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.79035394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13702024-02-13T21:39:53.838Z
13712024-02-13T21:39:53.838Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790393695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13722024-02-13T21:39:53.838Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.79040747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13732024-02-13T21:39:53.839Z{"msg":"9628627b-f4d5-42a8-99c8-fd9856e04289 is now active with session: a5d22715-6630-412c-b0d2-606a4f238b7c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790428713Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13742024-02-13T21:39:53.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790441718Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183,"session_id":"a5d22715-6630-412c-b0d2-606a4f238b7c"}
13752024-02-13T21:39:53.839Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:48.790460551Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
13762024-02-13T21:39:53.839ZLoading write count information from file "/tmp/test_repair_verify.data"
13772024-02-13T21:39:53.839ZError: Difference in extent metadata found!
13782024-02-13T21:39:53.839ZRun Repair workload
13792024-02-13T21:39:53.840Z01/30 Write block 375 len 40960 data: 3 2 2 2 2 2 3 3 3 3
13802024-02-13T21:39:53.840Z02/30 Write block 516 len 16384 data: 2 3 3 3
13812024-02-13T21:39:53.840Z03/30 Read block 23 len 20480
13822024-02-13T21:39:53.840Z04/30 Read block 120 len 20480
13832024-02-13T21:39:53.840Z05/30 Read block 503 len 32768
13842024-02-13T21:39:53.840Z06/30 Flush
13852024-02-13T21:39:53.840Z07/30 Write block 378 len 20480 data: 3 3 3 4 4
13862024-02-13T21:39:53.840Z08/30 Read block 46 len 16384
13872024-02-13T21:39:53.840Z09/30 Flush
13882024-02-13T21:39:53.840Z10/30 Write block 427 len 4096 data: 2
13892024-02-13T21:39:53.841Z11/30 Read block 53 len 8192
13902024-02-13T21:39:53.841Z12/30 Read block 314 len 20480
13912024-02-13T21:39:53.841Z13/30 Write block 54 len 40960 data: 2 2 2 2 2 2 2 2 2 2
13922024-02-13T21:39:53.841Z14/30 Read block 242 len 24576
13932024-02-13T21:39:53.841Z15/30 Read block 509 len 36864
13942024-02-13T21:39:53.841Z16/30 Flush
13952024-02-13T21:39:53.841Z17/30 Write block 18 len 20480 data: 4 4 4 4 4
13962024-02-13T21:39:53.841Z18/30 Read block 56 len 24576
13972024-02-13T21:39:53.841Z19/30 Flush
13982024-02-13T21:39:53.841Z20/30 Write block 585 len 16384 data: 3 3 3 2
13992024-02-13T21:39:53.841Z21/30 Flush
14002024-02-13T21:39:53.841Z22/30 Write block 56 len 40960 data: 3 3 3 3 3 3 3 3 2 2
14012024-02-13T21:39:53.841Z23/30 Write block 347 len 8192 data: 3 3
14022024-02-13T21:39:53.841Z24/30 Write block 249 len 40960 data: 2 2 2 2 2 2 2 2 2 2
14032024-02-13T21:39:53.842Z25/30 Write block 226 len 4096 data: 3
14042024-02-13T21:39:53.872Z26/30 Flush
14052024-02-13T21:39:53.872Z27/30 Write block 331 len 24576 data: 2 2 2 2 3 3
14062024-02-13T21:39:53.872Z28/30 Read block 295 len 24576
14072024-02-13T21:39:53.872Z29/30 Write block 58 len 24576 data: 4 4 4 4 4 4
14082024-02-13T21:39:53.872Z30/30 Write block 544 len 40960 data: 3 3 3 2 2 2 3 4 3 3
14092024-02-13T21:39:53.872Z----------------------------------------------------------------
14102024-02-13T21:39:53.872Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14112024-02-13T21:39:53.872ZStates: Active Active Active
14122024-02-13T21:39:53.873ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14132024-02-13T21:39:53.873Z 1 Acked 1000 Write 10 Done Done Done false
14142024-02-13T21:39:53.873Z 2 Acked 1001 Write 4 Done Done Sent false
14152024-02-13T21:39:53.873Z 3 Acked 1002 Read 5 Done Done Sent false
14162024-02-13T21:39:53.873Z 4 Acked 1003 Read 5 Done Done Sent false
14172024-02-13T21:39:53.873Z 5 Acked 1004 Read 8 Done Done Sent false
14182024-02-13T21:39:53.873Z 6 Acked 1005 Flush 0 Done Done Sent false
14192024-02-13T21:39:53.873Z 7 Acked 1006 Write 5 Done Done Sent false
14202024-02-13T21:39:53.873Z 8 Acked 1007 Read 4 Done Done Sent false
14212024-02-13T21:39:53.873Z 9 Acked 1008 Flush 0 Done Done Sent false
14222024-02-13T21:39:53.873Z 10 Acked 1009 Write 1 Done Done Sent false
14232024-02-13T21:39:53.873Z 11 Acked 1010 Read 2 Done Done Sent false
14242024-02-13T21:39:53.873Z 12 Acked 1011 Read 5 Done Done Sent false
14252024-02-13T21:39:53.874Z 13 Acked 1012 Write 10 Done Done Sent false
14262024-02-13T21:39:53.874Z 14 Acked 1013 Read 6 Done Done Sent false
14272024-02-13T21:39:53.874Z 15 Acked 1014 Read 9 Done Done Sent false
14282024-02-13T21:39:53.874Z 16 Acked 1015 Flush 0 Done Done Sent false
14292024-02-13T21:39:53.874Z 17 Acked 1016 Write 5 Done Done Sent false
14302024-02-13T21:39:53.874Z 18 Acked 1017 Read 6 Done Done Sent false
14312024-02-13T21:39:53.874Z 19 Acked 1018 Flush 0 Done Done Sent false
14322024-02-13T21:39:53.874Z 20 Acked 1019 Write 4 Done Done Sent false
14332024-02-13T21:39:53.874Z 21 Acked 1020 Flush 0 Done Done Sent false
14342024-02-13T21:39:53.874Z 22 Acked 1021 Write 10 Done Done Sent false
14352024-02-13T21:39:53.875Z 23 Acked 1022 Write 2 Done Done Sent false
14362024-02-13T21:39:53.875Z 24 Acked 1023 Write 10 Done Done Sent false
14372024-02-13T21:39:53.875Z 25 Acked 1024 Write 1 Done Done Sent false
14382024-02-13T21:39:53.875Z 26 Acked 1025 Flush 0 Done Done Sent false
14392024-02-13T21:39:53.875Z 27 Acked 1026 Write 6 Done Done Sent false
14402024-02-13T21:39:53.875Z 28 Acked 1027 Read 6 Done Done Sent false
14412024-02-13T21:39:53.875Z 29 Acked 1028 Write 6 Done Done Sent false
14422024-02-13T21:39:53.875Z 30 Acked 1029 Write 10 Sent Sent Sent false
14432024-02-13T21:39:53.875Z STATES DS:0 DS:1 DS:2 TOTAL
14442024-02-13T21:39:53.875Z New 0 0 0 0
14452024-02-13T21:39:53.875Z Sent 1 1 29 31
14462024-02-13T21:39:53.875Z Done 29 29 1 59
14472024-02-13T21:39:53.875Z Skipped 0 0 0 0
14482024-02-13T21:39:53.875Z Error 0 0 0 0
14492024-02-13T21:39:53.875ZLast Flush: 1025 1025 0
14502024-02-13T21:39:53.876ZDownstairs last five completed:
14512024-02-13T21:39:53.876ZUpstairs last five completed: 30 29 28 27 26
14522024-02-13T21:39:53.876Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:48.824695063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1183}
14532024-02-13T21:39:53.876ZWrote out file "/tmp/test_repair_verify.data"
14542024-02-13T21:39:53.876Z
14552024-02-13T21:39:53.876ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14562024-02-13T21:39:53.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14572024-02-13T21:39:53.876Z 0 000-019 12 12 9 23 23 17 F F F <---
14582024-02-13T21:39:53.876Z 1 020-039 12 12 11 23 23 19 F F F <---
14592024-02-13T21:39:53.876Z 2 040-059 12 12 8 25 25 15 T T F <---
14602024-02-13T21:39:53.876Z 3 060-079 12 12 5 25 25 9 T T F <---
14612024-02-13T21:39:53.876Z 4 080-099 7 7 7 13 13 13 F F F
14622024-02-13T21:39:53.876Z 5 100-119 1 1 1 1 1 1 F F F
14632024-02-13T21:39:53.876Z 6 120-139 9 9 9 17 17 17 F F F
14642024-02-13T21:39:53.877Z 7 140-159 3 3 3 4 4 4 F F F
14652024-02-13T21:39:53.877Z 8 160-179 5 5 5 9 9 9 F F F
14662024-02-13T21:39:53.877Z 9 180-199 7 7 7 13 13 13 F F F
14672024-02-13T21:39:53.877Z 10 200-219 11 11 11 19 19 19 F F F
14682024-02-13T21:39:53.877Z 11 220-239 12 12 11 25 25 19 F F F <---
14692024-02-13T21:39:53.877Z 12 240-259 12 12 11 25 25 19 F F F <---
14702024-02-13T21:39:53.877Z 13 260-279 2 2 2 2 2 2 F F F
14712024-02-13T21:39:53.877Z 14 280-299 7 7 7 13 13 13 F F F
14722024-02-13T21:39:53.877Z 15 300-319 11 11 11 19 19 19 F F F
14732024-02-13T21:39:53.877Z 16 320-339 9 9 9 17 17 17 T T F <---
14742024-02-13T21:39:53.877Z 17 340-359 12 12 11 25 25 19 F F F <---
14752024-02-13T21:39:53.878Z 18 360-379 12 12 11 21 21 19 F F T <---
14762024-02-13T21:39:53.878Z 19 380-399 12 12 2 21 21 2 F F T <---
14772024-02-13T21:39:53.878Z 20 400-419 11 11 11 19 19 19 F F F
14782024-02-13T21:39:53.878Z 21 420-439 12 12 10 22 22 18 F F F <---
14792024-02-13T21:39:53.878Z 22 440-459 10 10 10 18 18 18 F F F
14802024-02-13T21:39:53.878Z 23 460-479 7 7 7 13 13 13 F F F
14812024-02-13T21:39:53.878Z 24 480-499 8 8 8 16 16 16 F F F
14822024-02-13T21:39:53.878Z 25 500-519 12 12 6 20 20 12 F F F <---
14832024-02-13T21:39:53.878Z 26 520-539 9 9 9 17 17 17 F F F
14842024-02-13T21:39:53.878Z 27 540-559 9 9 9 17 17 17 T T F <---
14852024-02-13T21:39:53.878Z 28 560-579 8 8 8 16 16 16 F F F
14862024-02-13T21:39:53.881Z 29 580-599 12 12 8 24 24 16 F F F <---
14872024-02-13T21:39:53.882ZMax gen: 12, Max flush: 25
14882024-02-13T21:39:53.882ZA Difference in extent metadata is expected here
14892024-02-13T21:39:53.882ZOn loop 6
14902024-02-13T21:39:53.882Z
14912024-02-13T21:39:53.882ZVerifying data now
14922024-02-13T21:39:55.665ZNo change after verify
14932024-02-13T21:39:55.665ZLoop: 6 Downstairs dump after verify (and repair):
14942024-02-13T21:39:55.686ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14952024-02-13T21:39:55.686Z 0 000-019 12 12 12 23 23 23 F F F
14962024-02-13T21:39:55.686Z 1 020-039 12 12 12 23 23 23 F F F
14972024-02-13T21:39:55.686Z 2 040-059 13 13 13 26 26 26 F F F
14982024-02-13T21:39:55.696Z 3 060-079 13 13 13 26 26 26 F F F
14992024-02-13T21:39:55.696Z 4 080-099 7 7 7 13 13 13 F F F
15002024-02-13T21:39:55.696Z 5 100-119 1 1 1 1 1 1 F F F
15012024-02-13T21:39:55.697Z 6 120-139 9 9 9 17 17 17 F F F
15022024-02-13T21:39:55.697Z 7 140-159 3 3 3 4 4 4 F F F
15032024-02-13T21:39:55.697Z 8 160-179 5 5 5 9 9 9 F F F
15042024-02-13T21:39:55.697Z 9 180-199 7 7 7 13 13 13 F F F
15052024-02-13T21:39:55.697Z 10 200-219 11 11 11 19 19 19 F F F
15062024-02-13T21:39:55.697Z 11 220-239 12 12 12 25 25 25 F F F
15072024-02-13T21:39:55.697Z 12 240-259 12 12 12 25 25 25 F F F
15082024-02-13T21:39:55.697Z 13 260-279 2 2 2 2 2 2 F F F
15092024-02-13T21:39:55.698Z 14 280-299 7 7 7 13 13 13 F F F
15102024-02-13T21:39:55.698Z 15 300-319 11 11 11 19 19 19 F F F
15112024-02-13T21:39:55.698Z 16 320-339 13 13 13 26 26 26 F F F
15122024-02-13T21:39:55.698Z 17 340-359 12 12 12 25 25 25 F F F
15132024-02-13T21:39:55.698Z 18 360-379 12 12 12 21 21 21 F F F
15142024-02-13T21:39:55.698Z 19 380-399 12 12 12 21 21 21 F F F
15152024-02-13T21:39:55.698Z 20 400-419 11 11 11 19 19 19 F F F
15162024-02-13T21:39:55.698Z 21 420-439 12 12 12 22 22 22 F F F
15172024-02-13T21:39:55.698Z 22 440-459 10 10 10 18 18 18 F F F
15182024-02-13T21:39:55.699Z 23 460-479 7 7 7 13 13 13 F F F
15192024-02-13T21:39:55.699Z 24 480-499 8 8 8 16 16 16 F F F
15202024-02-13T21:39:55.699Z 25 500-519 12 12 12 20 20 20 F F F
15212024-02-13T21:39:55.699Z 26 520-539 9 9 9 17 17 17 F F F
15222024-02-13T21:39:55.699Z 27 540-559 13 13 13 26 26 26 F F F
15232024-02-13T21:39:55.699Z 28 560-579 8 8 8 16 16 16 F F F
15242024-02-13T21:39:55.699Z 29 580-599 12 12 12 24 24 24 F F F
15252024-02-13T21:39:55.699ZMax gen: 13, Max flush: 26
15262024-02-13T21:39:55.699Z
15272024-02-13T21:39:55.699ZBegin loop 7
15282024-02-13T21:39:55.699ZDownstairs to restart: 0
15292024-02-13T21:39:55.721Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540440477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
15302024-02-13T21:39:55.721Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540715837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
15312024-02-13T21:39:55.721ZCrucible runtime is spawned
15322024-02-13T21:39:55.721Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540784752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
15332024-02-13T21:39:55.722Z{"msg":"Crucible 2434730b-b91c-44ce-9ead-6a829bd20f2f has session id: 3c795e7d-34d0-4d86-8747-6cdc3dd84043","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540812702Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15342024-02-13T21:39:55.722Z{"msg":"Upstairs opts: Upstairs UUID: 2434730b-b91c-44ce-9ead-6a829bd20f2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540855906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15352024-02-13T21:39:55.722Z{"msg":"Crucible stats registered with UUID: 2434730b-b91c-44ce-9ead-6a829bd20f2f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540874409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15362024-02-13T21:39:55.722Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540890963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
15372024-02-13T21:39:55.722Z{"msg":"2434730b-b91c-44ce-9ead-6a829bd20f2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540905578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15382024-02-13T21:39:55.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540921792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15392024-02-13T21:39:55.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540940265Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15402024-02-13T21:39:55.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.540984519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15412024-02-13T21:39:55.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541017237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15422024-02-13T21:39:55.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541050105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15432024-02-13T21:39:55.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541098888Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15442024-02-13T21:39:55.722Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541132575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15452024-02-13T21:39:55.722Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541199211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15462024-02-13T21:39:55.722Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54123209Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15472024-02-13T21:39:55.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541255511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15482024-02-13T21:39:55.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541288489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15492024-02-13T21:39:55.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54131338Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15502024-02-13T21:39:55.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.541962414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15512024-02-13T21:39:55.723Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.542033229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15522024-02-13T21:39:55.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54208599Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15532024-02-13T21:39:55.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54211282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15542024-02-13T21:39:55.723Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.542506957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15552024-02-13T21:39:55.723Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.542550731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15562024-02-13T21:39:55.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54285669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15572024-02-13T21:39:55.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54288448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15582024-02-13T21:39:55.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543091135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15592024-02-13T21:39:55.724Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543127782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15602024-02-13T21:39:55.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543143296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15612024-02-13T21:39:55.724Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543164758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15622024-02-13T21:39:55.724Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543195677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15632024-02-13T21:39:55.724Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543210272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15642024-02-13T21:39:55.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543447146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15652024-02-13T21:39:55.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543837394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15662024-02-13T21:39:55.725Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543878179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15672024-02-13T21:39:55.725Z{"msg":"[0]R flush_numbers[0..12]: [23, 23, 26, 26, 13, 1, 17, 4, 9, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543894853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15682024-02-13T21:39:55.725Z{"msg":"[0]R generation[0..12]: [12, 12, 13, 13, 7, 1, 9, 3, 5, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543916646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15692024-02-13T21:39:55.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543946245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15702024-02-13T21:39:55.725Z{"msg":"[1]R flush_numbers[0..12]: [23, 23, 26, 26, 13, 1, 17, 4, 9, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543961449Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15712024-02-13T21:39:55.725Z{"msg":"[1]R generation[0..12]: [12, 12, 13, 13, 7, 1, 9, 3, 5, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.543992228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15722024-02-13T21:39:55.725Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544045729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairsWait for a query_work_queue command to finish before sending IO
15732024-02-13T21:39:55.726Z","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15742024-02-13T21:39:55.726Z{"msg":"[2]R flush_numbers[0..12]: [23, 23, 26, 26, 13, 1, 17, 4, 9, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544067131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15752024-02-13T21:39:55.726Z{"msg":"[2]R generation[0..12]: [12, 12, 13, 13, 7, 1, 9, 3, 5, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.54409794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15762024-02-13T21:39:55.726Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544114174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15772024-02-13T21:39:55.726Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544144783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15782024-02-13T21:39:55.727Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544160587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15792024-02-13T21:39:55.727Z{"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544175002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15802024-02-13T21:39:55.727Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544204541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15812024-02-13T21:39:55.727Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544219296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15822024-02-13T21:39:55.727Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544238139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15832024-02-13T21:39:55.727Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544257852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15842024-02-13T21:39:55.727Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544287701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15852024-02-13T21:39:55.727Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544302536Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15862024-02-13T21:39:55.727Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15872024-02-13T21:39:55.728Z2024-02-13T21:39:54.544321989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15882024-02-13T21:39:55.728Z{"msg":"2434730b-b91c-44ce-9ead-6a829bd20f2f is now active with session: 3c795e7d-34d0-4d86-8747-6cdc3dd84043","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544347389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15892024-02-13T21:39:55.728Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.5443722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191,"session_id":"3c795e7d-34d0-4d86-8747-6cdc3dd84043"}
15902024-02-13T21:39:55.728Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:54.544385686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
15912024-02-13T21:39:55.728ZLoading write count information from file "/tmp/test_repair_verify.data"
15922024-02-13T21:39:55.728ZRun Repair workload
15932024-02-13T21:39:55.728Z01/30 Write block 177 len 4096 data: 2
15942024-02-13T21:39:55.728Z02/30 Read block 262 len 28672
15952024-02-13T21:39:55.748Z03/30 Read block 238 len 28672
15962024-02-13T21:39:55.748Z04/30 Flush
15972024-02-13T21:39:55.749Z05/30 Write block 375 len 28672 data: 4 3 3 4 4 4 5
15982024-02-13T21:39:55.749Z06/30 Read block 571 len 32768
15992024-02-13T21:39:55.749Z07/30 Read block 159 len 20480
16002024-02-13T21:39:55.749Z08/30 Write block 109 len 32768 data: 2 2 2 2 2 2 2 2
16012024-02-13T21:39:55.749Z09/30 Write block 472 len 16384 data: 2 2 2 3
16022024-02-13T21:39:55.749Z10/30 Write block 526 len 20480 data: 2 2 2 2 2
16032024-02-13T21:39:55.749Z11/30 Read block 180 len 8192
16042024-02-13T21:39:55.750Z12/30 Write block 491 len 20480 data: 2 2 2 4 3
16052024-02-13T21:39:55.750Z13/30 Read block 548 len 12288
16062024-02-13T21:39:55.750Z14/30 Read block 29 len 20480
16072024-02-13T21:39:55.750Z15/30 Read block 4 len 16384
16082024-02-13T21:39:55.750Z16/30 Write block 520 len 24576 data: 3 3 3 2 2 2
16092024-02-13T21:39:55.750Z17/30 Read block 177 len 40960
16102024-02-13T21:39:55.751Z18/30 Read block 187 len 32768
16112024-02-13T21:39:55.751Z19/30 Flush
16122024-02-13T21:39:55.751Z20/30 Write block 11 len 36864 data: 2 2 2 2 3 4 4 5 5
16132024-02-13T21:39:55.751Z21/30 Read block 85 len 32768
16142024-02-13T21:39:55.751Z22/30 Write block 564 len 20480 data: 2 2 2 2 3
16152024-02-13T21:39:55.751Z23/30 Read block 340 len 16384
16162024-02-13T21:39:55.751Z24/30 Write block 541 len 8192 data: 3 2
16172024-02-13T21:39:55.751Z25/30 Read block 496 len 20480
16182024-02-13T21:39:55.752Z26/30 Read block 352 len 16384
16192024-02-13T21:39:55.752Z27/30 Write block 175 len 8192 data: 2 2
16202024-02-13T21:39:55.752Z28/30 Write block 445 len 40960 data: 2 2 2 2 2 2 3 3 3 3
16212024-02-13T21:39:55.752Z29/30 Read block 466 len 8192
16222024-02-13T21:39:55.752Z30/30 Read block 388 len 4096
16232024-02-13T21:39:55.752Z----------------------------------------------------------------
16242024-02-13T21:39:55.752Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16252024-02-13T21:39:55.752ZStates: Active Active Active
16262024-02-13T21:39:55.752ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16272024-02-13T21:39:55.752Z 1 Acked 1000 Write 1 Sent Done Done false
16282024-02-13T21:39:55.752Z 2 Acked 1001 Read 7 Sent Done Done false
16292024-02-13T21:39:55.752Z 3 Acked 1002 Read 7 Sent Done Done false
16302024-02-13T21:39:55.753Z 4 Acked 1003 Flush 0 Sent Done Done false
16312024-02-13T21:39:55.753Z 5 Acked 1004 Write 7 Sent Done Done false
16322024-02-13T21:39:55.753Z 6 Acked 1005 Read 8 Sent Done Done false
16332024-02-13T21:39:55.753Z 7 Acked 1006 Read 5 Sent Done Done false
16342024-02-13T21:39:55.753Z 8 Acked 1007 Write 8 Sent Done Done false
16352024-02-13T21:39:55.753Z 9 Acked 1008 Write 4 Sent Done Done false
16362024-02-13T21:39:55.753Z 10 Acked 1009 Write 5 Sent Done Done false
16372024-02-13T21:39:55.753Z 11 Acked 1010 Read 2 Sent Done Done false
16382024-02-13T21:39:55.753Z 12 Acked 1011 Write 5 Sent Done Done false
16392024-02-13T21:39:55.753Z 13 Acked 1012 Read 3 Sent Done Done false
16402024-02-13T21:39:55.753Z 14 Acked 1013 Read 5 Sent Done Done false
16412024-02-13T21:39:55.753Z 15 Acked 1014 Read 4 Sent Done Done false
16422024-02-13T21:39:55.754Z 16 Acked 1015 Write 6 Sent Done Done false
16432024-02-13T21:39:55.754Z 17 Acked 1016 Read 10 Sent Done Done false
16442024-02-13T21:39:55.754Z 18 Acked 1017 Read 8 Sent Done Done false
16452024-02-13T21:39:55.754Z 19 Acked 1018 Flush 0 Sent Done Done false
16462024-02-13T21:39:55.754Z 20 Acked 1019 Write 9 Sent Done Done false
16472024-02-13T21:39:55.754Z 21 Acked 1020 Read 8 Sent Done Done false
16482024-02-13T21:39:55.754Z 22 Acked 1021 Write 5 Sent Done Done false
16492024-02-13T21:39:55.754Z 23 Acked 1022 Read 4 Sent Done Done false
16502024-02-13T21:39:55.754Z 24 Acked 1023 Write 2 Sent Done Done false
16512024-02-13T21:39:55.754Z 25 Acked 1024 Read 5 Sent Done Done false
16522024-02-13T21:39:55.754Z 26 Acked 1025 Read 4 Sent Done Done false
16532024-02-13T21:39:55.755Z 27 Acked 1026 Write 2 Sent Done Done false
16542024-02-13T21:39:55.755Z 28 Acked 1027 Write 10 Sent Done Done false
16552024-02-13T21:39:55.755Z 29 Acked 1028 Read 2 Sent Done Done false
16562024-02-13T21:39:55.755Z 30 Acked 1029 Read 1 Sent Sent Done false
16572024-02-13T21:39:55.755Z STATES DS:0 DS:1 DS:2 TOTAL
16582024-02-13T21:39:55.755Z New 0 0 0 0
16592024-02-13T21:39:55.755Z Sent 30 1 0 31
16602024-02-13T21:39:55.755Z Done 0 29 30 59
16612024-02-13T21:39:55.756Z Skipped 0 0 0 0
16622024-02-13T21:39:55.756Z Error 0 0 0 0
16632024-02-13T21:39:55.756ZLast Flush: 0 1018 1018
16642024-02-13T21:39:55.756ZDownstairs last five completed:
16652024-02-13T21:39:55.756ZUpstairs last five completed: 30 29 28 27 26
16662024-02-13T21:39:55.756Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:54.573902639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1191}
16672024-02-13T21:39:55.756ZWrote out file "/tmp/test_repair_verify.data"
16682024-02-13T21:39:55.775Z
16692024-02-13T21:39:55.775ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16702024-02-13T21:39:55.776ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16712024-02-13T21:39:55.776Z 0 000-019 12 12 12 23 23 23 F T T <---
16722024-02-13T21:39:55.776Z 1 020-039 12 12 12 23 23 23 F F F
16732024-02-13T21:39:55.776Z 2 040-059 13 13 13 26 26 26 F F F
16742024-02-13T21:39:55.776Z 3 060-079 13 13 13 26 26 26 F F F
16752024-02-13T21:39:55.776Z 4 080-099 7 7 7 13 13 13 F F F
16762024-02-13T21:39:55.776Z 5 100-119 1 14 14 1 28 28 F F F <---
16772024-02-13T21:39:55.777Z 6 120-139 9 9 9 17 17 17 F F F
16782024-02-13T21:39:55.777Z 7 140-159 3 3 3 4 4 4 F F F
16792024-02-13T21:39:55.777Z 8 160-179 5 14 14 9 27 27 F T T <---
16802024-02-13T21:39:55.777Z 9 180-199 7 7 7 13 13 13 F F F
16812024-02-13T21:39:55.777Z 10 200-219 11 11 11 19 19 19 F F F
16822024-02-13T21:39:55.777Z 11 220-239 12 12 12 25 25 25 F F F
16832024-02-13T21:39:55.777Z 12 240-259 12 12 12 25 25 25 F F F
16842024-02-13T21:39:55.777Z 13 260-279 2 2 2 2 2 2 F F F
16852024-02-13T21:39:55.777Z 14 280-299 7 7 7 13 13 13 F F F
16862024-02-13T21:39:55.777Z 15 300-319 11 11 11 19 19 19 F F F
16872024-02-13T21:39:55.778Z 16 320-339 13 13 13 26 26 26 F F F
16882024-02-13T21:39:55.778Z 17 340-359 12 12 12 25 25 25 F F F
16892024-02-13T21:39:55.778Z 18 360-379 12 14 14 21 28 28 F F F <---
16902024-02-13T21:39:55.778Z 19 380-399 12 14 14 21 28 28 F F F <---
16912024-02-13T21:39:55.778Z 20 400-419 11 11 11 19 19 19 F F F
16922024-02-13T21:39:55.778Z 21 420-439 12 12 12 22 22 22 F F F
16932024-02-13T21:39:55.779Z 22 440-459 10 10 10 18 18 18 F T T <---
16942024-02-13T21:39:55.779Z 23 460-479 7 14 14 13 28 28 F F F <---
16952024-02-13T21:39:55.779Z 24 480-499 8 14 14 16 28 28 F F F <---
16962024-02-13T21:39:55.779Z 25 500-519 12 12 12 20 20 20 F F F
16972024-02-13T21:39:55.780Z 26 520-539 9 14 14 17 28 28 F F F <---
16982024-02-13T21:39:55.780Z 27 540-559 13 13 13 26 26 26 F T T <---
16992024-02-13T21:39:55.780Z 28 560-579 8 8 8 16 16 16 F T T <---
17002024-02-13T21:39:55.780Z 29 580-599 12 12 12 24 24 24 F F F
17012024-02-13T21:39:55.780ZMax gen: 14, Max flush: 28
17022024-02-13T21:39:55.781ZError: Difference in extent metadata found!
17032024-02-13T21:39:55.781ZA Difference in extent metadata is expected here
17042024-02-13T21:39:55.781ZOn loop 7
17052024-02-13T21:39:55.781Z
17062024-02-13T21:39:55.782ZVerifying data now
17072024-02-13T21:40:00.309ZNo change after verify
17082024-02-13T21:40:00.310ZLoop: 7 Downstairs dump after verify (and repair):
17092024-02-13T21:40:00.324ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17102024-02-13T21:40:00.324Z 0 000-019 15 15 15 29 29 29 F F F
17112024-02-13T21:40:00.324Z 1 020-039 12 12 12 23 23 23 F F F
17122024-02-13T21:40:00.324Z 2 040-059 13 13 13 26 26 26 F F F
17132024-02-13T21:40:00.324Z 3 060-079 13 13 13 26 26 26 F F F
17142024-02-13T21:40:00.324Z 4 080-099 7 7 7 13 13 13 F F F
17152024-02-13T21:40:00.325Z 5 100-119 14 14 14 28 28 28 F F F
17162024-02-13T21:40:00.325Z 6 120-139 9 9 9 17 17 17 F F F
17172024-02-13T21:40:00.325Z 7 140-159 3 3 3 4 4 4 F F F
17182024-02-13T21:40:00.325Z 8 160-179 15 15 15 29 29 29 F F F
17192024-02-13T21:40:00.325Z 9 180-199 7 7 7 13 13 13 F F F
17202024-02-13T21:40:00.330Z 10 200-219 11 11 11 19 19 19 F F F
17212024-02-13T21:40:00.330Z 11 220-239 12 12 12 25 25 25 F F F
17222024-02-13T21:40:00.330Z 12 240-259 12 12 12 25 25 25 F F F
17232024-02-13T21:40:00.330Z 13 260-279 2 2 2 2 2 2 F F F
17242024-02-13T21:40:00.331Z 14 280-299 7 7 7 13 13 13 F F F
17252024-02-13T21:40:00.331Z 15 300-319 11 11 11 19 19 19 F F F
17262024-02-13T21:40:00.331Z 16 320-339 13 13 13 26 26 26 F F F
17272024-02-13T21:40:00.331Z 17 340-359 12 12 12 25 25 25 F F F
17282024-02-13T21:40:00.331Z 18 360-379 14 14 14 28 28 28 F F F
17292024-02-13T21:40:00.331Z 19 380-399 14 14 14 28 28 28 F F F
17302024-02-13T21:40:00.331Z 20 400-419 11 11 11 19 19 19 F F F
17312024-02-13T21:40:00.331Z 21 420-439 12 12 12 22 22 22 F F F
17322024-02-13T21:40:00.331Z 22 440-459 15 15 15 29 29 29 F F F
17332024-02-13T21:40:00.331Z 23 460-479 14 14 14 28 28 28 F F F
17342024-02-13T21:40:00.332Z 24 480-499 14 14 14 28 28 28 F F F
17352024-02-13T21:40:00.332Z 25 500-519 12 12 12 20 20 20 F F F
17362024-02-13T21:40:00.332Z 26 520-539 14 14 14 28 28 28 F F F
17372024-02-13T21:40:00.332Z 27 540-559 15 15 15 29 29 29 F F F
17382024-02-13T21:40:00.332Z 28 560-579 15 15 15 29 29 29 F F F
17392024-02-13T21:40:00.332Z 29 580-599 12 12 12 24 24 24 F F F
17402024-02-13T21:40:00.332ZMax gen: 15, Max flush: 29
17412024-02-13T21:40:00.333Z
17422024-02-13T21:40:00.333ZBegin loop 8
17432024-02-13T21:40:00.333ZDownstairs to restart: 1
17442024-02-13T21:40:00.346Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.1841356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
17452024-02-13T21:40:00.347Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18438318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
17462024-02-13T21:40:00.347Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184401373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
17472024-02-13T21:40:00.354Z{"msg":"Crucible 0e51a34a-5ec2-40ee-bdc6-65e9caf7fdf6 has session id: 0159be80-683a-4185-a322-e6337d848856","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184435131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17482024-02-13T21:40:00.354Z{"msg":"Crucible runtime is spawned
17492024-02-13T21:40:00.354ZUpstairs opts: Upstairs UUID: 0e51a34a-5ec2-40ee-bdc6-65e9caf7fdf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184477925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17502024-02-13T21:40:00.354Z{"msg":"Crucible stats registered with UUID: 0e51a34a-5ec2-40ee-bdc6-65e9caf7fdf6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184504226Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17512024-02-13T21:40:00.354Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184536754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
17522024-02-13T21:40:00.355Z{"msg":"0e51a34a-5ec2-40ee-bdc6-65e9caf7fdf6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184607938Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17532024-02-13T21:40:00.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184639467Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17542024-02-13T21:40:00.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184664868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17552024-02-13T21:40:00.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184696646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17562024-02-13T21:40:00.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184722347Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17572024-02-13T21:40:00.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184752796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17582024-02-13T21:40:00.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184777857Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17592024-02-13T21:40:00.356Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184818982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17602024-02-13T21:40:00.356Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18485103Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17612024-02-13T21:40:00.356Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184889616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17622024-02-13T21:40:00.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.184915287Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17632024-02-13T21:40:00.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185019059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17642024-02-13T21:40:00.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185062464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17652024-02-13T21:40:00.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185661406Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17662024-02-13T21:40:00.357Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185696623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17672024-02-13T21:40:00.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185712088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17682024-02-13T21:40:00.357Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.185726572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17692024-02-13T21:40:00.357Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186239876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17702024-02-13T21:40:00.357Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186407485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17712024-02-13T21:40:00.358Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186707157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17722024-02-13T21:40:00.358Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18675247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17732024-02-13T21:40:00.358Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186779191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17742024-02-13T21:40:00.358Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186861311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17752024-02-13T21:40:00.358Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186922789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17762024-02-13T21:40:00.358Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.186957516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17772024-02-13T21:40:00.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18708434Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17782024-02-13T21:40:00.359Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.187117088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17792024-02-13T21:40:00.359Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.187577081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17802024-02-13T21:40:00.359Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.187976296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17812024-02-13T21:40:00.359Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18802098Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17822024-02-13T21:40:00.359Z{"msg":"[0]R flush_numbers[0..12]: [29, 23, 26, 26, 13, 28, 17, 4, 29, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188047901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17832024-02-13T21:40:00.359Z{"msg":"[0]R generation[0..12]: [15, 12, 13, 13, 7, 14, 9, 3, 15, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188082778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17842024-02-13T21:40:00.359Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188118585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17852024-02-13T21:40:00.360Z{"msg":"[1]R flush_numbers[0..12]: [29, 23, 26, 26, 13, 28, 17, 4, 29, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188153862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17862024-02-13T21:40:00.360Z{"msg":"[1]R generation[0..12]: [15, 12, 13, 13, 7, 14, 9, 3, 15, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188188789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17872024-02-13T21:40:00.361ZWait for a query_work_queue command to finish before sending IO
17882024-02-13T21:40:00.361Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188233113Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17892024-02-13T21:40:00.361Z{"msg":"[2]R flush_numbers[0..12]: [29, 23, 26, 26, 13, 28, 17, 4, 29, 13, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188274768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17902024-02-13T21:40:00.361Z{"msg":"[2]R generation[0..12]: [15, 12, 13, 13, 7, 14, 9, 3, 15, 7, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188312374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17912024-02-13T21:40:00.361Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188348661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17922024-02-13T21:40:00.361Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188384648Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17932024-02-13T21:40:00.361Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188412778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17942024-02-13T21:40:00.361Z{"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188437539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17952024-02-13T21:40:00.361Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188472316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17962024-02-13T21:40:00.361Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188506174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
17972024-02-13T21:40:00.361Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188520729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17982024-02-13T21:40:00.362Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188543001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
17992024-02-13T21:40:00.362Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188556806Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"0159be80-683a-4185-a322-e6337d848856"}
18002024-02-13T21:40:00.362Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188580577Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
18012024-02-13T21:40:00.362Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188596801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
18022024-02-13T21:40:00.362Z{"msg":"0e51a34a-5ec2-40ee-bdc6-65e9caf7fdf6 is now active with session: 0159be80-683a-4185-a322-e6337d848856","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188610217Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
18032024-02-13T21:40:00.362Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.18862957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199,"session_id":"0159be80-683a-4185-a322-e6337d848856"}
18042024-02-13T21:40:00.362Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:39:59.188642315Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
18052024-02-13T21:40:00.367ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18062024-02-13T21:40:00.367ZLoading write count information from file "/tmp/test_repair_verify.data"
18072024-02-13T21:40:00.367ZRun Repair workload
18082024-02-13T21:40:00.367Z01/30 Write block 471 len 4096 data: 2
18092024-02-13T21:40:00.367Z02/30 Write block 154 len 20480 data: 2 2 2 2 2
18102024-02-13T21:40:00.368Z03/30 Read block 106 len 8192
18112024-02-13T21:40:00.368Z04/30 Write block 416 len 16384 data: 2 2 2 2
18122024-02-13T21:40:00.368Z05/30 Write block 549 len 4096 data: 3
18132024-02-13T21:40:00.368Z06/30 Read block 89 len 12288
18142024-02-13T21:40:00.368Z07/30 Write block 382 len 8192 data: 5 4
18152024-02-13T21:40:00.368Z08/30 Read block 203 len 4096
18162024-02-13T21:40:00.369Z09/30 Read block 243 len 4096
18172024-02-13T21:40:00.369Z10/30 Read block 372 len 8192
18182024-02-13T21:40:00.369Z11/30 Read block 341 len 16384
18192024-02-13T21:40:00.369Z12/30 Write block 146 len 28672 data: 2 2 2 2 2 2 2
18202024-02-13T21:40:00.369Z13/30 Write block 307 len 8192 data: 3 3
18212024-02-13T21:40:00.369Z14/30 Read block 94 len 8192
18222024-02-13T21:40:00.369Z15/30 Flush
18232024-02-13T21:40:00.370Z16/30 Write block 574 len 20480 data: 3 3 3 3 3
18242024-02-13T21:40:00.370Z17/30 Read block 289 len 28672
18252024-02-13T21:40:00.370Z18/30 Read block 13 len 4096
18262024-02-13T21:40:00.370Z19/30 Write block 438 len 36864 data: 2 2 2 2 2 2 2 3 3
18272024-02-13T21:40:00.370Z20/30 Read block 495 len 24576
18282024-02-13T21:40:00.374Z21/30 Read block 489 len 12288
18292024-02-13T21:40:00.374Z22/30 Read block 100 len 8192
18302024-02-13T21:40:00.374Z23/30 Write block 88 len 32768 data: 3 2 2 2 2 2 2 2
18312024-02-13T21:40:00.374Z24/30 Write block 65 len 32768 data: 3 2 2 3 3 3 3 3
18322024-02-13T21:40:00.375Z25/30 Flush
18332024-02-13T21:40:00.375Z26/30 Write block 564 len 20480 data: 3 3 3 3 4
18342024-02-13T21:40:00.379Z27/30 Write block 331 len 40960 data: 3 3 3 3 4 4 3 3 3 3
18352024-02-13T21:40:00.379Z28/30 Write block 188 len 32768 data: 2 2 2 2 2 2 2 2
18362024-02-13T21:40:00.379Z29/30 Write block 447 len 20480 data: 3 3 3 3 4
18372024-02-13T21:40:00.380Z30/30 Write block 151 len 4096 data: 3
18382024-02-13T21:40:00.380Z----------------------------------------------------------------
18392024-02-13T21:40:00.380Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18402024-02-13T21:40:00.380ZStates: Active Active Active
18412024-02-13T21:40:00.380ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18422024-02-13T21:40:00.380Z 1 Acked 1000 Write 1 Done Done Done false
18432024-02-13T21:40:00.380Z 2 Acked 1001 Write 5 Done Done Done false
18442024-02-13T21:40:00.381Z 3 Acked 1002 Read 2 Done Done Done false
18452024-02-13T21:40:00.381Z 4 Acked 1003 Write 4 Done Sent Done false
18462024-02-13T21:40:00.381Z 5 Acked 1004 Write 1 Done Sent Done false
18472024-02-13T21:40:00.381Z 6 Acked 1005 Read 3 Done Sent Done false
18482024-02-13T21:40:00.381Z 7 Acked 1006 Write 2 Done Sent Done false
18492024-02-13T21:40:00.381Z 8 Acked 1007 Read 1 Done Sent Done false
18502024-02-13T21:40:00.382Z 9 Acked 1008 Read 1 Done Sent Done false
18512024-02-13T21:40:00.382Z 10 Acked 1009 Read 2 Done Sent Done false
18522024-02-13T21:40:00.382Z 11 Acked 1010 Read 4 Done Sent Done false
18532024-02-13T21:40:00.382Z 12 Acked 1011 Write 7 Done Sent Done false
18542024-02-13T21:40:00.382Z 13 Acked 1012 Write 2 Done Sent Done false
18552024-02-13T21:40:00.382Z 14 Acked 1013 Read 2 Done Sent Done false
18562024-02-13T21:40:00.382Z 15 Acked 1014 Flush 0 Done Sent Done false
18572024-02-13T21:40:00.382Z 16 Acked 1015 Write 5 Done Sent Done false
18582024-02-13T21:40:00.383Z 17 Acked 1016 Read 7 Done Sent Done false
18592024-02-13T21:40:00.383Z 18 Acked 1017 Read 1 Done Sent Done false
18602024-02-13T21:40:00.383Z 19 Acked 1018 Write 9 Done Sent Done false
18612024-02-13T21:40:00.383Z 20 Acked 1019 Read 6 Done Sent Done false
18622024-02-13T21:40:00.383Z 21 Acked 1020 Read 3 Done Sent Done false
18632024-02-13T21:40:00.383Z 22 Acked 1021 Read 2 Done Sent Done false
18642024-02-13T21:40:00.384Z 23 Acked 1022 Write 8 Done Sent Done false
18652024-02-13T21:40:00.384Z 24 Acked 1023 Write 8 Done Sent Done false
18662024-02-13T21:40:00.384Z 25 Acked 1024 Flush 0 Done Sent Done false
18672024-02-13T21:40:00.384Z 26 Acked 1025 Write 5 Done Sent Done false
18682024-02-13T21:40:00.384Z 27 Acked 1026 Write 10 Done Sent Done false
18692024-02-13T21:40:00.384Z 28 Acked 1027 Write 8 Done Sent Done false
18702024-02-13T21:40:00.384Z 29 Acked 1028 Write 5 Sent Sent Sent false
18712024-02-13T21:40:00.385Z 30 Acked 1029 Write 1 Sent Sent Sent false
18722024-02-13T21:40:00.385Z STATES DS:0 DS:1 DS:2 TOTAL
18732024-02-13T21:40:00.385Z New 0 0 0 0
18742024-02-13T21:40:00.385Z Sent 2 27 2 31
18752024-02-13T21:40:00.385Z Done 28 3 28 59
18762024-02-13T21:40:00.385Z Skipped 0 0 0 0
18772024-02-13T21:40:00.386Z Error 0 0 0 0
18782024-02-13T21:40:00.386ZLast Flush: 1024 0 1024
18792024-02-13T21:40:00.386ZDownstairs last five completed:
18802024-02-13T21:40:00.386ZUpstairs last five completed: 30 29 28 27 26
18812024-02-13T21:40:00.386Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:39:59.215547306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1199}
18822024-02-13T21:40:00.386ZWrote out file "/tmp/test_repair_verify.data"
18832024-02-13T21:40:00.391Z
18842024-02-13T21:40:00.391ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18852024-02-13T21:40:00.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18862024-02-13T21:40:00.401Z 0 000-019 15 15 15 29 29 29 F F F
18872024-02-13T21:40:00.401Z 1 020-039 12 12 12 23 23 23 F F F
18882024-02-13T21:40:00.402Z 2 040-059 13 13 13 26 26 26 F F F
18892024-02-13T21:40:00.402Z 3 060-079 16 13 16 31 26 31 F F F <---
18902024-02-13T21:40:00.402Z 4 080-099 16 7 16 31 13 31 F F F <---
18912024-02-13T21:40:00.402Z 5 100-119 14 14 14 28 28 28 F F F
18922024-02-13T21:40:00.402Z 6 120-139 9 9 9 17 17 17 F F F
18932024-02-13T21:40:00.407Z 7 140-159 16 3 16 30 4 30 T T T <---
18942024-02-13T21:40:00.407Z 8 160-179 15 15 15 29 29 29 F F F
18952024-02-13T21:40:00.407Z 9 180-199 7 7 7 13 13 13 T F T <---
18962024-02-13T21:40:00.408Z 10 200-219 11 11 11 19 19 19 F F F
18972024-02-13T21:40:00.408Z 11 220-239 12 12 12 25 25 25 F F F
18982024-02-13T21:40:00.408Z 12 240-259 12 12 12 25 25 25 F F F
18992024-02-13T21:40:00.408Z 13 260-279 2 2 2 2 2 2 F F F
19002024-02-13T21:40:00.408Z 14 280-299 7 7 7 13 13 13 F F F
19012024-02-13T21:40:00.408Z 15 300-319 16 11 16 30 19 30 F F F <---
19022024-02-13T21:40:00.408Z 16 320-339 13 13 13 26 26 26 T F T <---
19032024-02-13T21:40:00.408Z 17 340-359 12 12 12 25 25 25 T F T <---
19042024-02-13T21:40:00.408Z 18 360-379 14 14 14 28 28 28 F F F
19052024-02-13T21:40:00.409Z 19 380-399 16 14 16 30 28 30 F F F <---
19062024-02-13T21:40:00.409Z 20 400-419 16 11 16 30 19 30 F F F <---
19072024-02-13T21:40:00.409Z 21 420-439 16 12 16 31 22 31 F F F <---
19082024-02-13T21:40:00.409Z 22 440-459 16 15 16 31 29 31 T F T <---
19092024-02-13T21:40:00.409Z 23 460-479 16 14 16 30 28 30 F T F <---
19102024-02-13T21:40:00.409Z 24 480-499 14 14 14 28 28 28 F F F
19112024-02-13T21:40:00.409Z 25 500-519 12 12 12 20 20 20 F F F
19122024-02-13T21:40:00.409Z 26 520-539 14 14 14 28 28 28 F F F
19132024-02-13T21:40:00.410Z 27 540-559 16 15 16 30 29 30 F F F <---
19142024-02-13T21:40:00.410Z 28 560-579 16 15 16 31 29 31 T F T <---
19152024-02-13T21:40:00.410Z 29 580-599 12 12 12 24 24 24 F F F
19162024-02-13T21:40:00.410ZMax gen: 16, Max flush: 31
19172024-02-13T21:40:00.410ZError: Difference in extent metadata found!
19182024-02-13T21:40:00.410ZA Difference in extent metadata is expected here
19192024-02-13T21:40:00.410ZOn loop 8
19202024-02-13T21:40:00.410Z
19212024-02-13T21:40:00.410ZVerifying data now
19222024-02-13T21:40:06.489ZNo change after verify
19232024-02-13T21:40:06.511ZLoop: 8 Downstairs dump after verify (and repair):
19242024-02-13T21:40:06.512ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19252024-02-13T21:40:06.512Z 0 000-019 15 15 15 29 29 29 F F F
19262024-02-13T21:40:06.512Z 1 020-039 12 12 12 23 23 23 F F F
19272024-02-13T21:40:06.512Z 2 040-059 13 13 13 26 26 26 F F F
19282024-02-13T21:40:06.512Z 3 060-079 16 16 16 31 31 31 F F F
19292024-02-13T21:40:06.512Z 4 080-099 16 16 16 31 31 31 F F F
19302024-02-13T21:40:06.512Z 5 100-119 14 14 14 28 28 28 F F F
19312024-02-13T21:40:06.512Z 6 120-139 9 9 9 17 17 17 F F F
19322024-02-13T21:40:06.512Z 7 140-159 17 17 17 32 32 32 F F F
19332024-02-13T21:40:06.512Z 8 160-179 15 15 15 29 29 29 F F F
19342024-02-13T21:40:06.512Z 9 180-199 17 17 17 32 32 32 F F F
19352024-02-13T21:40:06.512Z 10 200-219 11 11 11 19 19 19 F F F
19362024-02-13T21:40:06.512Z 11 220-239 12 12 12 25 25 25 F F F
19372024-02-13T21:40:06.512Z 12 240-259 12 12 12 25 25 25 F F F
19382024-02-13T21:40:06.513Z 13 260-279 2 2 2 2 2 2 F F F
19392024-02-13T21:40:06.513Z 14 280-299 7 7 7 13 13 13 F F F
19402024-02-13T21:40:06.513Z 15 300-319 16 16 16 30 30 30 F F F
19412024-02-13T21:40:06.513Z 16 320-339 17 17 17 32 32 32 F F F
19422024-02-13T21:40:06.513Z 17 340-359 17 17 17 32 32 32 F F F
19432024-02-13T21:40:06.513Z 18 360-379 14 14 14 28 28 28 F F F
19442024-02-13T21:40:06.513Z 19 380-399 16 16 16 30 30 30 F F F
19452024-02-13T21:40:06.513Z 20 400-419 16 16 16 30 30 30 F F F
19462024-02-13T21:40:06.513Z 21 420-439 16 16 16 31 31 31 F F F
19472024-02-13T21:40:06.513Z 22 440-459 17 17 17 32 32 32 F F F
19482024-02-13T21:40:06.513Z 23 460-479 16 16 16 30 30 30 F F F
19492024-02-13T21:40:06.513Z 24 480-499 14 14 14 28 28 28 F F F
19502024-02-13T21:40:06.513Z 25 500-519 12 12 12 20 20 20 F F F
19512024-02-13T21:40:06.514Z 26 520-539 14 14 14 28 28 28 F F F
19522024-02-13T21:40:06.514Z 27 540-559 16 16 16 30 30 30 F F F
19532024-02-13T21:40:06.514Z 28 560-579 17 17 17 32 32 32 F F F
19542024-02-13T21:40:06.514Z 29 580-599 12 12 12 24 24 24 F F F
19552024-02-13T21:40:06.514ZMax gen: 17, Max flush: 32
19562024-02-13T21:40:06.514Z
19572024-02-13T21:40:06.514ZBegin loop 9
19582024-02-13T21:40:06.514ZDownstairs to restart: 0
19592024-02-13T21:40:06.514Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949213269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
19602024-02-13T21:40:06.514Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949472205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
19612024-02-13T21:40:06.514Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949492717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
19622024-02-13T21:40:06.514Z{Crucible runtime is spawned
19632024-02-13T21:40:06.514Z"msg":"Crucible 4720f810-3082-409f-90fd-42447c9ca2e1 has session id: 247960cc-88a5-40e2-a64e-ad95479d8211","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949542349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19642024-02-13T21:40:06.515Z{"msg":"Upstairs opts: Upstairs UUID: 4720f810-3082-409f-90fd-42447c9ca2e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949586243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19652024-02-13T21:40:06.515Z{"msg":"Crucible stats registered with UUID: 4720f810-3082-409f-90fd-42447c9ca2e1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949606246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19662024-02-13T21:40:06.515Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949635256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
19672024-02-13T21:40:06.515Z{"msg":"4720f810-3082-409f-90fd-42447c9ca2e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949668563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19682024-02-13T21:40:06.515Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949697283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19692024-02-13T21:40:06.515Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949716336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19702024-02-13T21:40:06.515Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949745386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19712024-02-13T21:40:06.515Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949763669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19722024-02-13T21:40:06.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949797917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19732024-02-13T21:40:06.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949818559Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19742024-02-13T21:40:06.516Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949847958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19752024-02-13T21:40:06.516Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949911026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19762024-02-13T21:40:06.516Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.949965846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19772024-02-13T21:40:06.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950022625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19782024-02-13T21:40:06.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950067559Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19792024-02-13T21:40:06.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950094839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19802024-02-13T21:40:06.516Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950812249Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19812024-02-13T21:40:06.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950868708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19822024-02-13T21:40:06.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950902896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19832024-02-13T21:40:06.517Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.950932135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19842024-02-13T21:40:06.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.95141644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19852024-02-13T21:40:06.517Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.951454416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19862024-02-13T21:40:06.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.951616597Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19872024-02-13T21:40:06.517Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.951669538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19882024-02-13T21:40:06.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.951975297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19892024-02-13T21:40:06.518Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952031317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19902024-02-13T21:40:06.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952059126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19912024-02-13T21:40:06.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952137698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19922024-02-13T21:40:06.518Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952164358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19932024-02-13T21:40:06.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952178993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19942024-02-13T21:40:06.518Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952359917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19952024-02-13T21:40:06.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952795179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19962024-02-13T21:40:06.518Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952841243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19972024-02-13T21:40:06.518Z{"msg":"[0]R flush_numbers[0..12]: [29, 23, 26, 31, 31, 28, 17, 32, 29, 32, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.95287683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19982024-02-13T21:40:06.518Z{"msg":"[0]R generation[0..12]: [15, 12, 13, 16, 16, 14, 9, 17, 15, 17, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952912717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
19992024-02-13T21:40:06.518Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952948354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20002024-02-13T21:40:06.518Z{"msg":"[1]R flush_numbers[0..12]: [29, 23, 26, 31, 31, 28, 17, 32, 29, 32, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.952985051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20012024-02-13T21:40:06.518ZWait for a query_work_queue command to finish before sending IO
20022024-02-13T21:40:06.518Z{"msg":"[1]R generation[0..12]: [15, 12, 13, 16, 16, 14, 9, 17, 15, 17, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953044629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20032024-02-13T21:40:06.518Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953086594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20042024-02-13T21:40:06.518Z{"msg":"[2]R flush_numbers[0..12]: [29, 23, 26, 31, 31, 28, 17, 32, 29, 32, 19, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953128719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20052024-02-13T21:40:06.518Z{"msg":"[2]R generation[0..12]: [15, 12, 13, 16, 16, 14, 9, 17, 15, 17, 11, 12]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953161716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20062024-02-13T21:40:06.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953190616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20072024-02-13T21:40:06.519Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953228532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20082024-02-13T21:40:06.519Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953267658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20092024-02-13T21:40:06.519Z{"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953307754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20102024-02-13T21:40:06.519Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.95334384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20112024-02-13T21:40:06.519Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953380757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20122024-02-13T21:40:06.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953417244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20132024-02-13T21:40:06.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953434078Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20142024-02-13T21:40:06.519Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20152024-02-13T21:40:06.519Z,"time":"2024-02-13T21:40:04.95345515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20162024-02-13T21:40:06.519Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953470854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20172024-02-13T21:40:06.519Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953490067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20182024-02-13T21:40:06.519Z{"msg":"4720f810-3082-409f-90fd-42447c9ca2e1 is now active with session: 247960cc-88a5-40e2-a64e-ad95479d8211","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953503712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20192024-02-13T21:40:06.519Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953516458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207,"session_id":"247960cc-88a5-40e2-a64e-ad95479d8211"}
20202024-02-13T21:40:06.519Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:04.953528843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
20212024-02-13T21:40:06.520ZLoading write count information from file "/tmp/test_repair_verify.data"
20222024-02-13T21:40:06.520ZRun Repair workload
20232024-02-13T21:40:06.537Z01/30 Write block 366 len 8192 data: 2 2
20242024-02-13T21:40:06.537Z02/30 Read block 153 len 8192
20252024-02-13T21:40:06.537Z03/30 Read block 87 len 16384
20262024-02-13T21:40:06.537Z04/30 Write block 264 len 36864 data: 3 3 3 3 3 2 2 2 2
20272024-02-13T21:40:06.537Z05/30 Read block 564 len 32768
20282024-02-13T21:40:06.537Z06/30 Read block 377 len 4096
20292024-02-13T21:40:06.538Z07/30 Write block 124 len 24576 data: 3 4 5 5 3 3
20302024-02-13T21:40:06.538Z08/30 Write block 11 len 32768 data: 3 3 3 3 4 5 5 6
20312024-02-13T21:40:06.538Z09/30 Read block 241 len 4096
20322024-02-13T21:40:06.538Z10/30 Read block 2 len 36864
20332024-02-13T21:40:06.538Z11/30 Flush
20342024-02-13T21:40:06.538Z12/30 Write block 196 len 32768 data: 2 2 3 3 3 3 2 2
20352024-02-13T21:40:06.538Z13/30 Read block 153 len 36864
20362024-02-13T21:40:06.539Z14/30 Write block 183 len 36864 data: 5 4 4 3 2 3 3 3 3
20372024-02-13T21:40:06.539Z15/30 Read block 378 len 4096
20382024-02-13T21:40:06.539Z16/30 Read block 327 len 24576
20392024-02-13T21:40:06.539Z17/30 Read block 271 len 24576
20402024-02-13T21:40:06.539Z18/30 Read block 544 len 8192
20412024-02-13T21:40:06.539Z19/30 Write block 499 len 8192 data: 3 3
20422024-02-13T21:40:06.539Z20/30 Flush
20432024-02-13T21:40:06.539Z21/30 Write block 192 len 20480 data: 3 3 3 3 3
20442024-02-13T21:40:06.540Z22/30 Read block 109 len 40960
20452024-02-13T21:40:06.540Z23/30 Write block 467 len 36864 data: 2 2 2 2 3 3 3 3 4
20462024-02-13T21:40:06.540Z24/30 Write block 217 len 24576 data: 4 4 4 3 3 3
20472024-02-13T21:40:06.540Z25/30 Write block 520 len 24576 data: 4 4 4 3 3 3
20482024-02-13T21:40:06.540Z26/30 Write block 364 len 32768 data: 2 2 3 3 2 2 2 2
20492024-02-13T21:40:06.540Z27/30 Write block 70 len 20480 data: 4 4 4 2 3
20502024-02-13T21:40:06.540Z28/30 Write block 493 len 36864 data: 3 5 4 3 3 4 4 4 2
20512024-02-13T21:40:06.541Z29/30 Write block 216 len 20480 data: 4 5 5 5 4
20522024-02-13T21:40:06.541Z30/30 Write block 509 len 28672 data: 4 3 3 3 3 3 2
20532024-02-13T21:40:06.541Z----------------------------------------------------------------
20542024-02-13T21:40:06.541Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20552024-02-13T21:40:06.541ZStates: Active Active Active
20562024-02-13T21:40:06.541ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20572024-02-13T21:40:06.541Z 1 Acked 1000 Write 2 Done Done Done false
20582024-02-13T21:40:06.541Z 2 Acked 1001 Read 2 Done Done Done false
20592024-02-13T21:40:06.541Z 3 Acked 1002 Read 4 Done Done Done false
20602024-02-13T21:40:06.542Z 4 Acked 1003 Write 9 Done Done Done false
20612024-02-13T21:40:06.542Z 5 Acked 1004 Read 8 Done Done Done false
20622024-02-13T21:40:06.542Z 6 Acked 1005 Read 1 Done Done Done false
20632024-02-13T21:40:06.542Z 7 Acked 1006 Write 6 Done Done Done false
20642024-02-13T21:40:06.542Z 8 Acked 1007 Write 8 Done Done Done false
20652024-02-13T21:40:06.542Z 9 Acked 1008 Read 1 Done Done Done false
20662024-02-13T21:40:06.542Z 10 Acked 1009 Read 9 Done Done Done false
20672024-02-13T21:40:06.542Z 11 Acked 1010 Flush 0 Sent Done Done false
20682024-02-13T21:40:06.542Z 12 Acked 1011 Write 8 Sent Done Done false
20692024-02-13T21:40:06.542Z 13 Acked 1012 Read 9 Sent Done Done false
20702024-02-13T21:40:06.542Z 14 Acked 1013 Write 9 Sent Done Done false
20712024-02-13T21:40:06.542Z 15 Acked 1014 Read 1 Sent Done Done false
20722024-02-13T21:40:06.543Z 16 Acked 1015 Read 6 Sent Done Done false
20732024-02-13T21:40:06.543Z 17 Acked 1016 Read 6 Sent Done Done false
20742024-02-13T21:40:06.543Z 18 Acked 1017 Read 2 Sent Done Done false
20752024-02-13T21:40:06.543Z 19 Acked 1018 Write 2 Sent Done Done false
20762024-02-13T21:40:06.543Z 20 Acked 1019 Flush 0 Sent Done Done false
20772024-02-13T21:40:06.543Z 21 Acked 1020 Write 5 Sent Done Done false
20782024-02-13T21:40:06.543Z 22 Acked 1021 Read 10 Sent Done Done false
20792024-02-13T21:40:06.543Z 23 Acked 1022 Write 9 Sent Done Done false
20802024-02-13T21:40:06.543Z 24 Acked 1023 Write 6 Sent Done Done false
20812024-02-13T21:40:06.543Z 25 Acked 1024 Write 6 Sent Done Done false
20822024-02-13T21:40:06.543Z 26 Acked 1025 Write 8 Sent Done Done false
20832024-02-13T21:40:06.543Z 27 Acked 1026 Write 5 Sent Done Done false
20842024-02-13T21:40:06.543Z 28 Acked 1027 Write 9 Sent Done Done false
20852024-02-13T21:40:06.544Z 29 Acked 1028 Write 5 Sent Done Done false
20862024-02-13T21:40:06.544Z 30 Acked 1029 Write 7 Sent Sent Sent false
20872024-02-13T21:40:06.544Z STATES DS:0 DS:1 DS:2 TOTAL
20882024-02-13T21:40:06.544Z New 0 0 0 0
20892024-02-13T21:40:06.544Z Sent 20 1 1 22
20902024-02-13T21:40:06.544Z Done 10 29 29 68
20912024-02-13T21:40:06.544Z Skipped 0 0 0 0
20922024-02-13T21:40:06.544Z Error 0 0 0 0
20932024-02-13T21:40:06.544ZLast Flush: 0 1019 1019
20942024-02-13T21:40:06.544ZDownstairs last five completed:
20952024-02-13T21:40:06.544ZUpstairs last five completed: 30 29 28 27 26
20962024-02-13T21:40:06.545Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:04.983880184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1207}
20972024-02-13T21:40:06.545ZWrote out file "/tmp/test_repair_verify.data"
20982024-02-13T21:40:06.545Z
20992024-02-13T21:40:06.545ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21002024-02-13T21:40:06.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21012024-02-13T21:40:06.545Z 0 000-019 15 18 18 29 33 33 T F F <---
21022024-02-13T21:40:06.545Z 1 020-039 12 12 12 23 23 23 F F F
21032024-02-13T21:40:06.545Z 2 040-059 13 13 13 26 26 26 F F F
21042024-02-13T21:40:06.545Z 3 060-079 16 16 16 31 31 31 F T T <---
21052024-02-13T21:40:06.545Z 4 080-099 16 16 16 31 31 31 F F F
21062024-02-13T21:40:06.545Z 5 100-119 14 14 14 28 28 28 F F F
21072024-02-13T21:40:06.545Z 6 120-139 9 18 18 17 33 33 T F F <---
21082024-02-13T21:40:06.545Z 7 140-159 17 17 17 32 32 32 F F F
21092024-02-13T21:40:06.545Z 8 160-179 15 15 15 29 29 29 F F F
21102024-02-13T21:40:06.545Z 9 180-199 17 18 18 32 34 34 F T T <---
21112024-02-13T21:40:06.545Z 10 200-219 11 18 18 19 34 34 F T T <---
21122024-02-13T21:40:06.545Z 11 220-239 12 12 12 25 25 25 F T T <---
21132024-02-13T21:40:06.545Z 12 240-259 12 12 12 25 25 25 F F F
21142024-02-13T21:40:06.545Z 13 260-279 2 18 18 2 33 33 T F F <---
21152024-02-13T21:40:06.545Z 14 280-299 7 7 7 13 13 13 F F F
21162024-02-13T21:40:06.546Z 15 300-319 16 16 16 30 30 30 F F F
21172024-02-13T21:40:06.546Z 16 320-339 17 17 17 32 32 32 F F F
21182024-02-13T21:40:06.546Z 17 340-359 17 17 17 32 32 32 F F F
21192024-02-13T21:40:06.546Z 18 360-379 14 18 18 28 33 33 T T T <---
21202024-02-13T21:40:06.546Z 19 380-399 16 16 16 30 30 30 F F F
21212024-02-13T21:40:06.546Z 20 400-419 16 16 16 30 30 30 F F F
21222024-02-13T21:40:06.546Z 21 420-439 16 16 16 31 31 31 F F F
21232024-02-13T21:40:06.557Z 22 440-459 17 17 17 32 32 32 F F F
21242024-02-13T21:40:06.557Z 23 460-479 16 16 16 30 30 30 F T T <---
21252024-02-13T21:40:06.557Z 24 480-499 14 18 18 28 34 34 F T T <---
21262024-02-13T21:40:06.557Z 25 500-519 12 18 18 20 34 34 F T T <---
21272024-02-13T21:40:06.557Z 26 520-539 14 14 14 28 28 28 F T T <---
21282024-02-13T21:40:06.557Z 27 540-559 16 16 16 30 30 30 F F F
21292024-02-13T21:40:06.557Z 28 560-579 17 17 17 32 32 32 F F F
21302024-02-13T21:40:06.558Z 29 580-599 12 12 12 24 24 24 F F F
21312024-02-13T21:40:06.558ZMax gen: 18, Max flush: 34
21322024-02-13T21:40:06.558ZError: Difference in extent metadata found!
21332024-02-13T21:40:06.558ZA Difference in extent metadata is expected here
21342024-02-13T21:40:06.558ZOn loop 9
21352024-02-13T21:40:06.558Z
21362024-02-13T21:40:06.558ZVerifying data now
21372024-02-13T21:40:11.778ZNo change after verify
21382024-02-13T21:40:11.778ZLoop: 9 Downstairs dump after verify (and repair):
21392024-02-13T21:40:11.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21402024-02-13T21:40:11.792Z 0 000-019 18 18 18 33 33 33 F F F
21412024-02-13T21:40:11.793Z 1 020-039 12 12 12 23 23 23 F F F
21422024-02-13T21:40:11.793Z 2 040-059 13 13 13 26 26 26 F F F
21432024-02-13T21:40:11.803Z 3 060-079 19 19 19 35 35 35 F F F
21442024-02-13T21:40:11.804Z 4 080-099 16 16 16 31 31 31 F F F
21452024-02-13T21:40:11.804Z 5 100-119 14 14 14 28 28 28 F F F
21462024-02-13T21:40:11.804Z 6 120-139 18 18 18 33 33 33 F F F
21472024-02-13T21:40:11.804Z 7 140-159 17 17 17 32 32 32 F F F
21482024-02-13T21:40:11.805Z 8 160-179 15 15 15 29 29 29 F F F
21492024-02-13T21:40:11.805Z 9 180-199 19 19 19 35 35 35 F F F
21502024-02-13T21:40:11.805Z 10 200-219 19 19 19 35 35 35 F F F
21512024-02-13T21:40:11.805Z 11 220-239 19 19 19 35 35 35 F F F
21522024-02-13T21:40:11.806Z 12 240-259 12 12 12 25 25 25 F F F
21532024-02-13T21:40:11.806Z 13 260-279 18 18 18 33 33 33 F F F
21542024-02-13T21:40:11.806Z 14 280-299 7 7 7 13 13 13 F F F
21552024-02-13T21:40:11.806Z 15 300-319 16 16 16 30 30 30 F F F
21562024-02-13T21:40:11.806Z 16 320-339 17 17 17 32 32 32 F F F
21572024-02-13T21:40:11.807Z 17 340-359 17 17 17 32 32 32 F F F
21582024-02-13T21:40:11.807Z 18 360-379 19 19 19 35 35 35 F F F
21592024-02-13T21:40:11.807Z 19 380-399 16 16 16 30 30 30 F F F
21602024-02-13T21:40:11.807Z 20 400-419 16 16 16 30 30 30 F F F
21612024-02-13T21:40:11.807Z 21 420-439 16 16 16 31 31 31 F F F
21622024-02-13T21:40:11.808Z 22 440-459 17 17 17 32 32 32 F F F
21632024-02-13T21:40:11.808Z 23 460-479 19 19 19 35 35 35 F F F
21642024-02-13T21:40:11.808Z 24 480-499 19 19 19 35 35 35 F F F
21652024-02-13T21:40:11.808Z 25 500-519 19 19 19 35 35 35 F F F
21662024-02-13T21:40:11.808Z 26 520-539 19 19 19 35 35 35 F F F
21672024-02-13T21:40:11.808Z 27 540-559 16 16 16 30 30 30 F F F
21682024-02-13T21:40:11.809Z 28 560-579 17 17 17 32 32 32 F F F
21692024-02-13T21:40:11.809Z 29 580-599 12 12 12 24 24 24 F F F
21702024-02-13T21:40:11.809ZMax gen: 19, Max flush: 35
21712024-02-13T21:40:11.809Z
21722024-02-13T21:40:11.810ZBegin loop 10
21732024-02-13T21:40:11.810ZDownstairs to restart: 2
21742024-02-13T21:40:11.821Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.645918309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
21752024-02-13T21:40:11.821Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.64619182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
21762024-02-13T21:40:11.821Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646215851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
21772024-02-13T21:40:11.821Z{"msg":"Crucible ed02615f-7c35-4e7f-9d2d-875ebf612a66 has session id: 8801db24-9d21-4b72-a981-4eb2c726b44d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646236843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21782024-02-13T21:40:11.822Z{"msg":"Upstairs opts: Upstairs UUID: ed02615f-7c35-4e7f-9d2d-875ebf612a66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646258685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21792024-02-13T21:40:11.822Z{"msg":"Crucible stats registered with UUID: ed02615f-7c35-4e7f-9d2d-875ebf612a66","v":0,"name":"crucible","level":30Crucible runtime is spawned
21802024-02-13T21:40:11.822Z,"time":"2024-02-13T21:40:10.646282627Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21812024-02-13T21:40:11.822Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646380591Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
21822024-02-13T21:40:11.822Z{"msg":"ed02615f-7c35-4e7f-9d2d-875ebf612a66 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646483634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21832024-02-13T21:40:11.822Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646510154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21842024-02-13T21:40:11.822Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646525348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21852024-02-13T21:40:11.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646557617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21862024-02-13T21:40:11.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646589545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21872024-02-13T21:40:11.823Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646615466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21882024-02-13T21:40:11.823Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646640127Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21892024-02-13T21:40:11.823Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646663298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21902024-02-13T21:40:11.823Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646693867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21912024-02-13T21:40:11.824Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646720108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21922024-02-13T21:40:11.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646835216Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21932024-02-13T21:40:11.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.64688053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21942024-02-13T21:40:11.824Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.646899283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21952024-02-13T21:40:11.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.647616173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21962024-02-13T21:40:11.825Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.647666055Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21972024-02-13T21:40:11.825Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.647708669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21982024-02-13T21:40:11.825Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.647739138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
21992024-02-13T21:40:11.825Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648160006Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22002024-02-13T21:40:11.825Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648275174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22012024-02-13T21:40:11.825Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648400758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22022024-02-13T21:40:11.825Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648430977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22032024-02-13T21:40:11.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648634423Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22042024-02-13T21:40:11.836Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648688594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22052024-02-13T21:40:11.836Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648716164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22062024-02-13T21:40:11.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648752151Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22072024-02-13T21:40:11.836Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.648775722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22082024-02-13T21:40:11.836Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.6488101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22092024-02-13T21:40:11.837Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649032029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22102024-02-13T21:40:11.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.64944407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22112024-02-13T21:40:11.837Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649482646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22122024-02-13T21:40:11.838Z{"msg":"[0]R flush_numbers[0..12]: [33, 23, 26, 35, 31, 28, 33, 32, 29, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649520392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22132024-02-13T21:40:11.838Z{"msg":"[0]R generation[0..12]: [18, 12, 13, 19, 16, 14, 18, 17, 15, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649549771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22142024-02-13T21:40:11.838Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649576522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22152024-02-13T21:40:11.838Z{"msg":"[1]R flush_numbers[0..12]: [33, 23, 26, 35, 31, 28, 33, 32, 29, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649605101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22162024-02-13T21:40:11.839Z{"msg":"[1]R generation[0..12]: [18, 12, 13, 19, 16, 14, 18, 17, 15, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649630172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22172024-02-13T21:40:11.839Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22182024-02-13T21:40:11.839Z,"time":"2024-02-13T21:40:10.64966339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22192024-02-13T21:40:11.839Z{"msg":"[2]R flush_numbers[0..12]: [33, 23, 26, 35, 31, 28, 33, 32, 29, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649689341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22202024-02-13T21:40:11.840Z{"msg":"[2]R generation[0..12]: [18, 12, 13, 19, 16, 14, 18, 17, 15, 19, 19, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649716951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22212024-02-13T21:40:11.840Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649742092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22222024-02-13T21:40:11.840Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649768462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22232024-02-13T21:40:11.840Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649791194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22242024-02-13T21:40:11.840Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649817974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22252024-02-13T21:40:11.841Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649840956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22262024-02-13T21:40:11.841Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649867476Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22272024-02-13T21:40:11.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649889318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22282024-02-13T21:40:11.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649917058Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22292024-02-13T21:40:11.841Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22302024-02-13T21:40:11.842Z,"time":"2024-02-13T21:40:10.64993989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22312024-02-13T21:40:11.842Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.649978146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22322024-02-13T21:40:11.842Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.650008855Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22332024-02-13T21:40:11.842Z{"msg":"ed02615f-7c35-4e7f-9d2d-875ebf612a66 is now active with session: 8801db24-9d21-4b72-a981-4eb2c726b44d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.650033356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22342024-02-13T21:40:11.842Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.650062285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215,"session_id":"8801db24-9d21-4b72-a981-4eb2c726b44d"}
22352024-02-13T21:40:11.842Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:10.650083688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
22362024-02-13T21:40:11.842ZLoading write count information from file "/tmp/test_repair_verify.data"
22372024-02-13T21:40:11.842ZRun Repair workload
22382024-02-13T21:40:11.842Z01/30 Write block 63 len 16384 data: 5 3 4 3
22392024-02-13T21:40:11.842Z02/30 Write block 418 len 36864 data: 3 3 2 2 2 2 2 2 2
22402024-02-13T21:40:11.843Z03/30 Read block 295 len 36864
22412024-02-13T21:40:11.843Z04/30 Write block 495 len 16384 data: 5 4 4 5
22422024-02-13T21:40:11.843Z05/30 Write block 102 len 8192 data: 2 2
22432024-02-13T21:40:11.843Z06/30 Read block 101 len 8192
22442024-02-13T21:40:11.843Z07/30 Write block 95 len 8192 data: 3 3
22452024-02-13T21:40:11.843Z08/30 Write block 428 len 12288 data: 2 2 2
22462024-02-13T21:40:11.844Z09/30 Read block 249 len 20480
22472024-02-13T21:40:11.844Z10/30 Read block 23 len 4096
22482024-02-13T21:40:11.844Z11/30 Read block 237 len 16384
22492024-02-13T21:40:11.844Z12/30 Write block 208 len 40960 data: 3 3 3 3 3 4 3 3 5 6
22502024-02-13T21:40:11.844Z13/30 Flush
22512024-02-13T21:40:11.845Z14/30 Write block 549 len 8192 data: 4 4
22522024-02-13T21:40:11.845Z15/30 Read block 326 len 8192
22532024-02-13T21:40:11.861Z16/30 Write block 307 len 24576 data: 4 4 3 3 3 3
22542024-02-13T21:40:11.861Z17/30 Read block 195 len 32768
22552024-02-13T21:40:11.861Z18/30 Read block 317 len 32768
22562024-02-13T21:40:11.861Z19/30 Write block 551 len 4096 data: 5
22572024-02-13T21:40:11.861Z20/30 Read block 82 len 28672
22582024-02-13T21:40:11.861Z21/30 Read block 465 len 36864
22592024-02-13T21:40:11.862Z22/30 Read block 532 len 24576
22602024-02-13T21:40:11.862Z23/30 Write block 17 len 40960 data: 6 7 6 5 5 5 2 2 2 2
22612024-02-13T21:40:11.862Z24/30 Write block 34 len 28672 data: 3 3 3 3 3 3 3
22622024-02-13T21:40:11.862Z25/30 Read block 197 len 16384
22632024-02-13T21:40:11.862Z26/30 Write block 185 len 40960 data: 5 4 3 4 4 4 4 4 4 4
22642024-02-13T21:40:11.862Z27/30 Read block 262 len 16384
22652024-02-13T21:40:11.862Z28/30 Read block 163 len 4096
22662024-02-13T21:40:11.863Z29/30 Read block 481 len 36864
22672024-02-13T21:40:11.863Z30/30 Write block 504 len 16384 data: 2 2 3 3
22682024-02-13T21:40:11.863Z----------------------------------------------------------------
22692024-02-13T21:40:11.863Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22702024-02-13T21:40:11.863ZStates: Active Active Active
22712024-02-13T21:40:11.863ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22722024-02-13T21:40:11.863Z 1 Acked 1000 Write 4 Done Done Sent false
22732024-02-13T21:40:11.863Z 2 Acked 1001 Write 9 Done Done Sent false
22742024-02-13T21:40:11.863Z 3 Acked 1002 Read 9 Done Done Sent false
22752024-02-13T21:40:11.864Z 4 Acked 1003 Write 4 Done Done Sent false
22762024-02-13T21:40:11.864Z 5 Acked 1004 Write 2 Done Done Sent false
22772024-02-13T21:40:11.864Z 6 Acked 1005 Read 2 Done Done Sent false
22782024-02-13T21:40:11.864Z 7 Acked 1006 Write 2 Done Done Sent false
22792024-02-13T21:40:11.864Z 8 Acked 1007 Write 3 Done Done Sent false
22802024-02-13T21:40:11.864Z 9 Acked 1008 Read 5 Done Done Sent false
22812024-02-13T21:40:11.864Z 10 Acked 1009 Read 1 Done Done Sent false
22822024-02-13T21:40:11.864Z 11 Acked 1010 Read 4 Done Done Sent false
22832024-02-13T21:40:11.865Z 12 Acked 1011 Write 10 Done Done Sent false
22842024-02-13T21:40:11.865Z 13 Acked 1012 Flush 0 Done Done Sent false
22852024-02-13T21:40:11.865Z 14 Acked 1013 Write 2 Done Done Sent false
22862024-02-13T21:40:11.865Z 15 Acked 1014 Read 2 Done Done Sent false
22872024-02-13T21:40:11.865Z 16 Acked 1015 Write 6 Done Done Sent false
22882024-02-13T21:40:11.865Z 17 Acked 1016 Read 8 Done Done Sent false
22892024-02-13T21:40:11.865Z 18 Acked 1017 Read 8 Done Done Sent false
22902024-02-13T21:40:11.865Z 19 Acked 1018 Write 1 Done Done Sent false
22912024-02-13T21:40:11.866Z 20 Acked 1019 Read 7 Done Done Sent false
22922024-02-13T21:40:11.866Z 21 Acked 1020 Read 9 Done Done Sent false
22932024-02-13T21:40:11.866Z 22 Acked 1021 Read 6 Done Done Sent false
22942024-02-13T21:40:11.866Z 23 Acked 1022 Write 10 Done Done Sent false
22952024-02-13T21:40:11.866Z 24 Acked 1023 Write 7 Done Done Sent false
22962024-02-13T21:40:11.866Z 25 Acked 1024 Read 4 Done Done Sent false
22972024-02-13T21:40:11.866Z 26 Acked 1025 Write 10 Done Done Sent false
22982024-02-13T21:40:11.866Z 27 Acked 1026 Read 4 Done Done Sent false
22992024-02-13T21:40:11.867Z 28 Acked 1027 Read 1 Done Done Sent false
23002024-02-13T21:40:11.867Z 29 Acked 1028 Read 9 Done Done Sent false
23012024-02-13T21:40:11.867Z 30 Acked 1029 Write 4 Sent Sent Sent false
23022024-02-13T21:40:11.867Z STATES DS:0 DS:1 DS:2 TOTAL
23032024-02-13T21:40:11.867Z New 0 0 0 0
23042024-02-13T21:40:11.867Z Sent 1 1 30 32
23052024-02-13T21:40:11.867Z Done 29 29 0 58
23062024-02-13T21:40:11.867Z Skipped 0 0 0 0
23072024-02-13T21:40:11.867Z Error 0 0 0 0
23082024-02-13T21:40:11.868ZLast Flush: 1012 1012 0
23092024-02-13T21:40:11.868ZDownstairs last five completed:
23102024-02-13T21:40:11.868ZUpstairs last five completed: 30 29 28 27 26
23112024-02-13T21:40:11.868Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:10.678405204Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1215}
23122024-02-13T21:40:11.868ZWrote out file "/tmp/test_repair_verify.data"
23132024-02-13T21:40:11.868Z
23142024-02-13T21:40:11.868ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23152024-02-13T21:40:11.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23162024-02-13T21:40:11.874Z 0 000-019 18 18 18 33 33 33 T T F <---
23172024-02-13T21:40:11.874Z 1 020-039 12 12 12 23 23 23 T T F <---
23182024-02-13T21:40:11.874Z 2 040-059 13 13 13 26 26 26 T T F <---
23192024-02-13T21:40:11.874Z 3 060-079 20 20 19 36 36 35 F F F <---
23202024-02-13T21:40:11.874Z 4 080-099 20 20 16 36 36 31 F F F <---
23212024-02-13T21:40:11.875Z 5 100-119 20 20 14 36 36 28 F F F <---
23222024-02-13T21:40:11.875Z 6 120-139 18 18 18 33 33 33 F F F
23232024-02-13T21:40:11.875Z 7 140-159 17 17 17 32 32 32 F F F
23242024-02-13T21:40:11.875Z 8 160-179 15 15 15 29 29 29 F F F
23252024-02-13T21:40:11.875Z 9 180-199 19 19 19 35 35 35 T T F <---
23262024-02-13T21:40:11.875Z 10 200-219 20 20 19 36 36 35 F F F <---
23272024-02-13T21:40:11.875Z 11 220-239 19 19 19 35 35 35 F F F
23282024-02-13T21:40:11.875Z 12 240-259 12 12 12 25 25 25 F F F
23292024-02-13T21:40:11.876Z 13 260-279 18 18 18 33 33 33 F F F
23302024-02-13T21:40:11.876Z 14 280-299 7 7 7 13 13 13 F F F
23312024-02-13T21:40:11.876Z 15 300-319 16 16 16 30 30 30 T T F <---
23322024-02-13T21:40:11.876Z 16 320-339 17 17 17 32 32 32 F F F
23332024-02-13T21:40:11.877Z 17 340-359 17 17 17 32 32 32 F F F
23342024-02-13T21:40:11.877Z 18 360-379 19 19 19 35 35 35 F F F
23352024-02-13T21:40:11.877Z 19 380-399 16 16 16 30 30 30 F F F
23362024-02-13T21:40:11.877Z 20 400-419 20 20 16 36 36 30 F F F <---
23372024-02-13T21:40:11.877Z 21 420-439 20 20 16 36 36 31 F F F <---
23382024-02-13T21:40:11.877Z 22 440-459 17 17 17 32 32 32 F F F
23392024-02-13T21:40:11.877Z 23 460-479 19 19 19 35 35 35 F F F
23402024-02-13T21:40:11.877Z 24 480-499 20 20 19 36 36 35 F F F <---
23412024-02-13T21:40:11.878Z 25 500-519 19 19 19 35 35 35 T T F <---
23422024-02-13T21:40:11.878Z 26 520-539 19 19 19 35 35 35 F F F
23432024-02-13T21:40:11.878Z 27 540-559 16 16 16 30 30 30 T T F <---
23442024-02-13T21:40:11.878Z 28 560-579 17 17 17 32 32 32 F F F
23452024-02-13T21:40:11.878Z 29 580-599 12 12 12 24 24 24 F F F
23462024-02-13T21:40:11.878ZMax gen: 20, Max flush: 36
23472024-02-13T21:40:11.879ZError: Difference in extent metadata found!
23482024-02-13T21:40:11.879ZA Difference in extent metadata is expected here
23492024-02-13T21:40:11.879ZOn loop 10
23502024-02-13T21:40:11.879Z
23512024-02-13T21:40:11.884ZVerifying data now
23522024-02-13T21:40:16.547ZNo change after verify
23532024-02-13T21:40:16.547ZLoop: 10 Downstairs dump after verify (and repair):
23542024-02-13T21:40:16.561ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23552024-02-13T21:40:16.561Z 0 000-019 21 21 21 37 37 37 F F F
23562024-02-13T21:40:16.562Z 1 020-039 21 21 21 37 37 37 F F F
23572024-02-13T21:40:16.562Z 2 040-059 21 21 21 37 37 37 F F F
23582024-02-13T21:40:16.565Z 3 060-079 20 20 20 36 36 36 F F F
23592024-02-13T21:40:16.565Z 4 080-099 20 20 20 36 36 36 F F F
23602024-02-13T21:40:16.565Z 5 100-119 20 20 20 36 36 36 F F F
23612024-02-13T21:40:16.565Z 6 120-139 18 18 18 33 33 33 F F F
23622024-02-13T21:40:16.565Z 7 140-159 17 17 17 32 32 32 F F F
23632024-02-13T21:40:16.565Z 8 160-179 15 15 15 29 29 29 F F F
23642024-02-13T21:40:16.565Z 9 180-199 21 21 21 37 37 37 F F F
23652024-02-13T21:40:16.565Z 10 200-219 20 20 20 36 36 36 F F F
23662024-02-13T21:40:16.565Z 11 220-239 19 19 19 35 35 35 F F F
23672024-02-13T21:40:16.565Z 12 240-259 12 12 12 25 25 25 F F F
23682024-02-13T21:40:16.565Z 13 260-279 18 18 18 33 33 33 F F F
23692024-02-13T21:40:16.565Z 14 280-299 7 7 7 13 13 13 F F F
23702024-02-13T21:40:16.566Z 15 300-319 21 21 21 37 37 37 F F F
23712024-02-13T21:40:16.566Z 16 320-339 17 17 17 32 32 32 F F F
23722024-02-13T21:40:16.566Z 17 340-359 17 17 17 32 32 32 F F F
23732024-02-13T21:40:16.566Z 18 360-379 19 19 19 35 35 35 F F F
23742024-02-13T21:40:16.566Z 19 380-399 16 16 16 30 30 30 F F F
23752024-02-13T21:40:16.566Z 20 400-419 20 20 20 36 36 36 F F F
23762024-02-13T21:40:16.566Z 21 420-439 20 20 20 36 36 36 F F F
23772024-02-13T21:40:16.566Z 22 440-459 17 17 17 32 32 32 F F F
23782024-02-13T21:40:16.566Z 23 460-479 19 19 19 35 35 35 F F F
23792024-02-13T21:40:16.566Z 24 480-499 20 20 20 36 36 36 F F F
23802024-02-13T21:40:16.566Z 25 500-519 21 21 21 37 37 37 F F F
23812024-02-13T21:40:16.566Z 26 520-539 19 19 19 35 35 35 F F F
23822024-02-13T21:40:16.566Z 27 540-559 21 21 21 37 37 37 F F F
23832024-02-13T21:40:16.566Z 28 560-579 17 17 17 32 32 32 F F F
23842024-02-13T21:40:16.566Z 29 580-599 12 12 12 24 24 24 F F F
23852024-02-13T21:40:16.566ZMax gen: 21, Max flush: 37
23862024-02-13T21:40:16.566Z
23872024-02-13T21:40:16.566ZBegin loop 11
23882024-02-13T21:40:16.566ZDownstairs to restart: 0
23892024-02-13T21:40:16.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414493165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
23902024-02-13T21:40:16.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414714765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
23912024-02-13T21:40:16.582Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414740325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
23922024-02-13T21:40:16.587Z{"msg":"Crucible 69d4e6bd-b7f3-41bc-af63-3d6bbd4cc465 has session id: 2f86f0d8-5ee0-4a98-9c85-4448eb839415","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414765187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
23932024-02-13T21:40:16.587Z{"msg":"Upstairs opts: Upstairs UUID: 69d4e6bd-b7f3-41bc-af63-3d6bbd4cc465, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414807251Z","hostname":"Crucible runtime is spawned
23942024-02-13T21:40:16.587Zip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
23952024-02-13T21:40:16.587Z{"msg":"Crucible stats registered with UUID: 69d4e6bd-b7f3-41bc-af63-3d6bbd4cc465","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414850196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
23962024-02-13T21:40:16.587Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.414928647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
23972024-02-13T21:40:16.588Z{"msg":"69d4e6bd-b7f3-41bc-af63-3d6bbd4cc465 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415025552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
23982024-02-13T21:40:16.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415053462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
23992024-02-13T21:40:16.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415069846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24002024-02-13T21:40:16.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415083911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24012024-02-13T21:40:16.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41511384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24022024-02-13T21:40:16.588Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415138021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24032024-02-13T21:40:16.588Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415164302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24042024-02-13T21:40:16.588Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415188203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24052024-02-13T21:40:16.588Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415231207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24062024-02-13T21:40:16.588Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415298753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24072024-02-13T21:40:16.588Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41533437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24082024-02-13T21:40:16.588Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415475369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24092024-02-13T21:40:16.588Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.415532688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24102024-02-13T21:40:16.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416063656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24112024-02-13T21:40:16.588Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416142127Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24122024-02-13T21:40:16.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416211852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24132024-02-13T21:40:16.588Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416265232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24142024-02-13T21:40:16.588Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416747357Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24152024-02-13T21:40:16.588Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.416786703Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24162024-02-13T21:40:16.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417026696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24172024-02-13T21:40:16.588Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417067001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24182024-02-13T21:40:16.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417296768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24192024-02-13T21:40:16.588Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417351898Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24202024-02-13T21:40:16.589Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417379068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24212024-02-13T21:40:16.589Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417489658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24222024-02-13T21:40:16.589Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417532752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24232024-02-13T21:40:16.589Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417587832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24242024-02-13T21:40:16.589Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.417615593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24252024-02-13T21:40:16.589Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41800821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24262024-02-13T21:40:16.589Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418044947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24272024-02-13T21:40:16.593Z{"msg":"[0]R flush_numbers[0..12]: [37, 37, 37, 36, 36, 36, 33, 32, 29, 37, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41811779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24282024-02-13T21:40:16.593Z{"msg":"[0]R generation[0..12]: [21, 21, 21, 20, 20, 20, 18, 17, 15, 21, 20, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41814598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24292024-02-13T21:40:16.594Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418161304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24302024-02-13T21:40:16.594Z{"msg":"[1]R flush_numbers[0..12]: [37, 37, 37, 36, 36, 36, 33, 32, 29, 37, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418176489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24312024-02-13T21:40:16.594Z{"msg":"[1]R generation[0..12]: [21, 21, 21, 20, 20, 20, 18, 17, 15, 21, 20, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418197082Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24322024-02-13T21:40:16.594Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418211836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24332024-02-13T21:40:16.594Z{"msg":"[2]R flush_numbers[0..12]: [37, 37, 37, 36, 36, 36, 33, 32, 29, 37, 36, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418226051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24342024-02-13T21:40:16.594Z{"msg":"[2]R generation[0..12]: [21, 21, 21, 20, 20, 20, 18, 17, 15, 21, 20, 19]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418246864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24352024-02-13T21:40:16.594Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418261108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24362024-02-13T21:40:16.594Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418274194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24372024-02-13T21:40:16.594ZWait for a query_work_queue command to finish before sending IO
24382024-02-13T21:40:16.594Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418301214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24392024-02-13T21:40:16.594Z{"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418318527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24402024-02-13T21:40:16.594Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.41833939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24412024-02-13T21:40:16.594Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418353175Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24422024-02-13T21:40:16.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418365141Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24432024-02-13T21:40:16.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418385513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24442024-02-13T21:40:16.594Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418399088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24452024-02-13T21:40:16.594Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418412543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24462024-02-13T21:40:16.594Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418425029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24472024-02-13T21:40:16.594Z{"msg":"69d4e6bd-b7f3-41bc-af63-3d6bbd4cc465 is now active with session: 2f86f0d8-5ee0-4a98-9c85-4448eb839415","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418446021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24482024-02-13T21:40:16.594Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418458327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223,"session_id":"2f86f0d8-5ee0-4a98-9c85-4448eb839415"}
24492024-02-13T21:40:16.594Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:15.418470782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
24502024-02-13T21:40:16.594ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24512024-02-13T21:40:16.594ZLoading write count information from file "/tmp/test_repair_verify.data"
24522024-02-13T21:40:16.595ZRun Repair workload
24532024-02-13T21:40:16.595Z01/30 Write block 379 len 40960 data: 5 5 6 6 5 4 3 3 3 2
24542024-02-13T21:40:16.595Z02/30 Write block 202 len 16384 data: 3 3 3 3
24552024-02-13T21:40:16.595Z03/30 Write block 26 len 4096 data: 3
24562024-02-13T21:40:16.595Z04/30 Write block 413 len 20480 data: 2 2 2 3 3
24572024-02-13T21:40:16.595Z05/30 Read block 245 len 32768
24582024-02-13T21:40:16.595Z06/30 Flush
24592024-02-13T21:40:16.598Z07/30 Write block 589 len 4096 data: 2
24602024-02-13T21:40:16.598Z08/30 Write block 589 len 36864 data: 3 3 3 3 3 3 3 3 3
24612024-02-13T21:40:16.599Z09/30 Read block 384 len 12288
24622024-02-13T21:40:16.599Z10/30 Read block 171 len 32768
24632024-02-13T21:40:16.599Z11/30 Write block 414 len 32768 data: 3 3 4 4 4 4 3 3
24642024-02-13T21:40:16.599Z12/30 Read block 142 len 12288
24652024-02-13T21:40:16.601Z13/30 Read block 174 len 40960
24662024-02-13T21:40:16.601Z14/30 Write block 297 len 16384 data: 3 2 2 2
24672024-02-13T21:40:16.602Z15/30 Read block 527 len 40960
24682024-02-13T21:40:16.606Z16/30 Write block 417 len 4096 data: 5
24692024-02-13T21:40:16.606Z17/30 Read block 545 len 40960
24702024-02-13T21:40:16.606Z18/30 Write block 234 len 20480 data: 2 2 2 2 3
24712024-02-13T21:40:16.606Z19/30 Read block 272 len 36864
24722024-02-13T21:40:16.609Z20/30 Write block 35 len 8192 data: 4 4
24732024-02-13T21:40:16.610Z21/30 Write block 441 len 36864 data: 3 3 3 3 4 4 4 4 4
24742024-02-13T21:40:16.610Z22/30 Write block 325 len 16384 data: 2 2 2 3
24752024-02-13T21:40:16.610Z23/30 Flush
24762024-02-13T21:40:16.612Z24/30 Write block 564 len 28672 data: 4 4 4 4 5 3 3
24772024-02-13T21:40:16.615Z25/30 Read block 401 len 12288
24782024-02-13T21:40:16.615Z26/30 Read block 119 len 24576
24792024-02-13T21:40:16.615Z27/30 Write block 482 len 32768 data: 2 2 2 2 2 2 2 2
24802024-02-13T21:40:16.622Z28/30 Read block 521 len 8192
24812024-02-13T21:40:16.622Z29/30 Write block 234 len 40960 data: 3 3 3 3 4 3 3 3 3 3
24822024-02-13T21:40:16.622Z30/30 Write block 197 len 24576 data: 3 4 4 4 4 4
24832024-02-13T21:40:16.622Z----------------------------------------------------------------
24842024-02-13T21:40:16.623Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24852024-02-13T21:40:16.623ZStates: Active Active Active
24862024-02-13T21:40:16.623ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24872024-02-13T21:40:16.623Z 1 Acked 1000 Write 10 Done Done Done false
24882024-02-13T21:40:16.623Z 2 Acked 1001 Write 4 Done Done Done false
24892024-02-13T21:40:16.623Z 3 Acked 1002 Write 1 Sent Done Done false
24902024-02-13T21:40:16.623Z 4 Acked 1003 Write 5 Sent Done Done false
24912024-02-13T21:40:16.623Z 5 Acked 1004 Read 8 Sent Done Done false
24922024-02-13T21:40:16.623Z 6 Acked 1005 Flush 0 Sent Done Done false
24932024-02-13T21:40:16.623Z 7 Acked 1006 Write 1 Sent Done Done false
24942024-02-13T21:40:16.623Z 8 Acked 1007 Write 9 Sent Done Done false
24952024-02-13T21:40:16.623Z 9 Acked 1008 Read 3 Sent Done Done false
24962024-02-13T21:40:16.623Z 10 Acked 1009 Read 8 Sent Done Done false
24972024-02-13T21:40:16.623Z 11 Acked 1010 Write 8 Sent Done Done false
24982024-02-13T21:40:16.623Z 12 Acked 1011 Read 3 Sent Done Done false
24992024-02-13T21:40:16.623Z 13 Acked 1012 Read 10 Sent Done Done false
25002024-02-13T21:40:16.623Z 14 Acked 1013 Write 4 Sent Done Done false
25012024-02-13T21:40:16.623Z 15 Acked 1014 Read 10 Sent Done Done false
25022024-02-13T21:40:16.623Z 16 Acked 1015 Write 1 Sent Done Done false
25032024-02-13T21:40:16.624Z 17 Acked 1016 Read 10 Sent Done Done false
25042024-02-13T21:40:16.624Z 18 Acked 1017 Write 5 Sent Done Done false
25052024-02-13T21:40:16.624Z 19 Acked 1018 Read 9 Sent Done Done false
25062024-02-13T21:40:16.624Z 20 Acked 1019 Write 2 Sent Done Done false
25072024-02-13T21:40:16.624Z 21 Acked 1020 Write 9 Sent Done Done false
25082024-02-13T21:40:16.624Z 22 Acked 1021 Write 4 Sent Done Done false
25092024-02-13T21:40:16.624Z 23 Acked 1022 Flush 0 Sent Done Done false
25102024-02-13T21:40:16.624Z 24 Acked 1023 Write 7 Sent Done Done false
25112024-02-13T21:40:16.624Z 25 Acked 1024 Read 3 Sent Done Done false
25122024-02-13T21:40:16.624Z 26 Acked 1025 Read 6 Sent Done Done false
25132024-02-13T21:40:16.624Z 27 Acked 1026 Write 8 Sent Done Done false
25142024-02-13T21:40:16.624Z 28 Acked 1027 Read 2 Sent Done Done false
25152024-02-13T21:40:16.624Z 29 Acked 1028 Write 10 Sent Sent Sent false
25162024-02-13T21:40:16.624Z 30 Acked 1029 Write 6 Sent Sent Sent false
25172024-02-13T21:40:16.624Z STATES DS:0 DS:1 DS:2 TOTAL
25182024-02-13T21:40:16.624Z New 0 0 0 0
25192024-02-13T21:40:16.624Z Sent 28 2 2 32
25202024-02-13T21:40:16.624Z Done 2 28 28 58
25212024-02-13T21:40:16.624Z Skipped 0 0 0 0
25222024-02-13T21:40:16.624Z Error 0 0 0 0
25232024-02-13T21:40:16.625ZLast Flush: 0 1022 1022
25242024-02-13T21:40:16.625ZDownstairs last five completed:
25252024-02-13T21:40:16.625ZUpstairs last five completed: 30 29 28 27 26
25262024-02-13T21:40:16.625Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:15.451186235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1223}
25272024-02-13T21:40:16.625ZWrote out file "/tmp/test_repair_verify.data"
25282024-02-13T21:40:16.630Z
25292024-02-13T21:40:16.630ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25302024-02-13T21:40:16.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25312024-02-13T21:40:16.650Z 0 000-019 21 21 21 37 37 37 F F F
25322024-02-13T21:40:16.650Z 1 020-039 21 22 22 37 39 39 F F F <---
25332024-02-13T21:40:16.650Z 2 040-059 21 21 21 37 37 37 F F F
25342024-02-13T21:40:16.650Z 3 060-079 20 20 20 36 36 36 F F F
25352024-02-13T21:40:16.650Z 4 080-099 20 20 20 36 36 36 F F F
25362024-02-13T21:40:16.650Z 5 100-119 20 20 20 36 36 36 F F F
25372024-02-13T21:40:16.650Z 6 120-139 18 18 18 33 33 33 F F F
25382024-02-13T21:40:16.658Z 7 140-159 17 17 17 32 32 32 F F F
25392024-02-13T21:40:16.659Z 8 160-179 15 15 15 29 29 29 F F F
25402024-02-13T21:40:16.659Z 9 180-199 21 21 21 37 37 37 F T T <---
25412024-02-13T21:40:16.659Z 10 200-219 20 22 22 36 38 38 T T T <---
25422024-02-13T21:40:16.659Z 11 220-239 19 22 22 35 39 39 F T T <---
25432024-02-13T21:40:16.659Z 12 240-259 12 12 12 25 25 25 F T T <---
25442024-02-13T21:40:16.659Z 13 260-279 18 18 18 33 33 33 F F F
25452024-02-13T21:40:16.659Z 14 280-299 7 22 22 13 39 39 F F F <---
25462024-02-13T21:40:16.659Z 15 300-319 21 22 22 37 39 39 F F F <---
25472024-02-13T21:40:16.659Z 16 320-339 17 22 22 32 39 39 F F F <---
25482024-02-13T21:40:16.659Z 17 340-359 17 17 17 32 32 32 F F F
25492024-02-13T21:40:16.659Z 18 360-379 19 22 22 35 38 38 T F F <---
25502024-02-13T21:40:16.659Z 19 380-399 16 22 22 30 38 38 T F F <---
25512024-02-13T21:40:16.659Z 20 400-419 20 22 22 36 39 39 F F F <---
25522024-02-13T21:40:16.659Z 21 420-439 20 22 22 36 39 39 F F F <---
25532024-02-13T21:40:16.659Z 22 440-459 17 22 22 32 39 39 F F F <---
25542024-02-13T21:40:16.659Z 23 460-479 19 19 19 35 35 35 F F F
25552024-02-13T21:40:16.659Z 24 480-499 20 20 20 36 36 36 F T T <---
25562024-02-13T21:40:16.659Z 25 500-519 21 21 21 37 37 37 F F F
25572024-02-13T21:40:16.660Z 26 520-539 19 19 19 35 35 35 F F F
25582024-02-13T21:40:16.660Z 27 540-559 21 21 21 37 37 37 F F F
25592024-02-13T21:40:16.660Z 28 560-579 17 17 17 32 32 32 F T T <---
25602024-02-13T21:40:16.660Z 29 580-599 12 22 22 24 39 39 F F F <---
25612024-02-13T21:40:16.660ZMax gen: 22, Max flush: 39
25622024-02-13T21:40:16.660ZError: Difference in extent metadata found!
25632024-02-13T21:40:16.660ZA Difference in extent metadata is expected here
25642024-02-13T21:40:16.660ZOn loop 11
25652024-02-13T21:40:16.660Z
25662024-02-13T21:40:16.660ZVerifying data now
25672024-02-13T21:40:22.402ZNo change after verify
25682024-02-13T21:40:22.402ZLoop: 11 Downstairs dump after verify (and repair):
25692024-02-13T21:40:22.417ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25702024-02-13T21:40:22.417Z 0 000-019 21 21 21 37 37 37 F F F
25712024-02-13T21:40:22.417Z 1 020-039 22 22 22 39 39 39 F F F
25722024-02-13T21:40:22.417Z 2 040-059 21 21 21 37 37 37 F F F
25732024-02-13T21:40:22.418Z 3 060-079 20 20 20 36 36 36 F F F
25742024-02-13T21:40:22.418Z 4 080-099 20 20 20 36 36 36 F F F
25752024-02-13T21:40:22.418Z 5 100-119 20 20 20 36 36 36 F F F
25762024-02-13T21:40:22.418Z 6 120-139 18 18 18 33 33 33 F F F
25772024-02-13T21:40:22.418Z 7 140-159 17 17 17 32 32 32 F F F
25782024-02-13T21:40:22.418Z 8 160-179 15 15 15 29 29 29 F F F
25792024-02-13T21:40:22.419Z 9 180-199 23 23 23 40 40 40 F F F
25802024-02-13T21:40:22.419Z 10 200-219 23 23 23 40 40 40 F F F
25812024-02-13T21:40:22.419Z 11 220-239 23 23 23 40 40 40 F F F
25822024-02-13T21:40:22.423Z 12 240-259 23 23 23 40 40 40 F F F
25832024-02-13T21:40:22.423Z 13 260-279 18 18 18 33 33 33 F F F
25842024-02-13T21:40:22.423Z 14 280-299 22 22 22 39 39 39 F F F
25852024-02-13T21:40:22.423Z 15 300-319 22 22 22 39 39 39 F F F
25862024-02-13T21:40:22.423Z 16 320-339 22 22 22 39 39 39 F F F
25872024-02-13T21:40:22.423Z 17 340-359 17 17 17 32 32 32 F F F
25882024-02-13T21:40:22.424Z 18 360-379 22 22 22 38 38 38 F F F
25892024-02-13T21:40:22.424Z 19 380-399 22 22 22 38 38 38 F F F
25902024-02-13T21:40:22.424Z 20 400-419 22 22 22 39 39 39 F F F
25912024-02-13T21:40:22.424Z 21 420-439 22 22 22 39 39 39 F F F
25922024-02-13T21:40:22.424Z 22 440-459 22 22 22 39 39 39 F F F
25932024-02-13T21:40:22.424Z 23 460-479 19 19 19 35 35 35 F F F
25942024-02-13T21:40:22.424Z 24 480-499 23 23 23 40 40 40 F F F
25952024-02-13T21:40:22.424Z 25 500-519 21 21 21 37 37 37 F F F
25962024-02-13T21:40:22.425Z 26 520-539 19 19 19 35 35 35 F F F
25972024-02-13T21:40:22.425Z 27 540-559 21 21 21 37 37 37 F F F
25982024-02-13T21:40:22.425Z 28 560-579 23 23 23 40 40 40 F F F
25992024-02-13T21:40:22.425Z 29 580-599 22 22 22 39 39 39 F F F
26002024-02-13T21:40:22.425ZMax gen: 23, Max flush: 40
26012024-02-13T21:40:22.425Z
26022024-02-13T21:40:22.425ZBegin loop 12
26032024-02-13T21:40:22.425ZDownstairs to restart: 0
26042024-02-13T21:40:22.448Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.266671259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
26052024-02-13T21:40:22.448Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.266927666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
26062024-02-13T21:40:22.455Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.266965533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
26072024-02-13T21:40:22.455Z{"msg":"Crucible runtime is spawned
26082024-02-13T21:40:22.455ZCrucible aaec7a6c-1b89-4f85-95c8-c14710ac42a3 has session id: dd7311ca-554d-4fef-bb94-710de41dba76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267025721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26092024-02-13T21:40:22.455Z{"msg":"Upstairs opts: Upstairs UUID: aaec7a6c-1b89-4f85-95c8-c14710ac42a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267079421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26102024-02-13T21:40:22.455Z{"msg":"Crucible stats registered with UUID: aaec7a6c-1b89-4f85-95c8-c14710ac42a3","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267120347Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26112024-02-13T21:40:22.456Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267153874Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
26122024-02-13T21:40:22.456Z{"msg":"aaec7a6c-1b89-4f85-95c8-c14710ac42a3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267273381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26132024-02-13T21:40:22.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.26730448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26142024-02-13T21:40:22.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267346075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26152024-02-13T21:40:22.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.2673864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26162024-02-13T21:40:22.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267410751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26172024-02-13T21:40:22.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.26747009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26182024-02-13T21:40:22.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267499229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26192024-02-13T21:40:22.456Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.26752339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26202024-02-13T21:40:22.457Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267555709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26212024-02-13T21:40:22.457Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267581699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26222024-02-13T21:40:22.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267605571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26232024-02-13T21:40:22.457Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:21.267646046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26242024-02-13T21:40:22.457Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:21.267686141Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26252024-02-13T21:40:22.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267713092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26262024-02-13T21:40:22.457Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:21.267864327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26272024-02-13T21:40:22.457Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.2679117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26282024-02-13T21:40:22.457Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.267990651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26292024-02-13T21:40:22.457Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268064874Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26302024-02-13T21:40:22.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268226155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26312024-02-13T21:40:22.458Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.26826974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26322024-02-13T21:40:22.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268317053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26332024-02-13T21:40:22.458Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268345812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26342024-02-13T21:40:22.458Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268850539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26352024-02-13T21:40:22.458Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.268944675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26362024-02-13T21:40:22.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269274335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26372024-02-13T21:40:22.458Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269355516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26382024-02-13T21:40:22.458Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269374349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26392024-02-13T21:40:22.459Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269395581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26402024-02-13T21:40:22.459Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269416943Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26412024-02-13T21:40:22.459Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:21.269430849Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26422024-02-13T21:40:23.439Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.268861489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26432024-02-13T21:40:23.439Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.268935262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26442024-02-13T21:40:23.444Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.269655251Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26452024-02-13T21:40:23.444Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.269695216Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26462024-02-13T21:40:23.444Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270162547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26472024-02-13T21:40:23.444Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270617122Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26482024-02-13T21:40:23.444Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270670552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26492024-02-13T21:40:23.444Z{"msg":"[0]R flush_numbers[0..12]: [37, 39, 37, 36, 36, 36, 33, 32, 29, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270692255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26502024-02-13T21:40:23.444Z{"msg":"[0]R generation[0..12]: [21, 22, 21, 20, 20, 20, 18, 17, 15, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270718945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26512024-02-13T21:40:23.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270737848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26522024-02-13T21:40:23.444Z{"msg":"[1]R flush_numbers[0..12]: [37, 39, 37, 36, 36, 36, 33, 32, 29, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270763639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26532024-02-13T21:40:23.444Z{"msg":"[1]R generation[0..12]: [21, 22, 21, 20, 20, 20, 18, 17, 15, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270781352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26542024-02-13T21:40:23.445Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270807463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26552024-02-13T21:40:23.445ZWait for a query_work_queue command to finish before sending IO
26562024-02-13T21:40:23.445Z{"msg":"[2]R flush_numbers[0..12]: [37, 39, 37, 36, 36, 36, 33, 32, 29, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270831194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26572024-02-13T21:40:23.445Z{"msg":"[2]R generation[0..12]: [21, 22, 21, 20, 20, 20, 18, 17, 15, 23, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270848998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26582024-02-13T21:40:23.445Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270866611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26592024-02-13T21:40:23.445Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270891232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26602024-02-13T21:40:23.445Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270908316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26612024-02-13T21:40:23.445Z{"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270932497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26622024-02-13T21:40:23.445Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270949271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26632024-02-13T21:40:23.445Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.270966245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26642024-02-13T21:40:23.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.27098131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26652024-02-13T21:40:23.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.2710076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26662024-02-13T21:40:23.445Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271025554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26672024-02-13T21:40:23.445Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271053144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26682024-02-13T21:40:23.445Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271069368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26692024-02-13T21:40:23.445Z{"msg":"aaec7a6c-1b89-4f85-95c8-c14710ac42a3 is now active with session: dd7311ca-554d-4fef-bb94-710de41dba76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271099937Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26702024-02-13T21:40:23.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271122279Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231,"session_id":"dd7311ca-554d-4fef-bb94-710de41dba76"}
26712024-02-13T21:40:23.445Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:22.271150948Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
26722024-02-13T21:40:23.445ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26732024-02-13T21:40:23.445ZLoading write count information from file "/tmp/test_repair_verify.data"
26742024-02-13T21:40:23.445ZRun Repair workload
26752024-02-13T21:40:23.445Z01/30 Flush
26762024-02-13T21:40:23.449Z02/30 Flush
26772024-02-13T21:40:23.449Z03/30 Write block 583 len 4096 data: 3
26782024-02-13T21:40:23.449Z04/30 Flush
26792024-02-13T21:40:23.449Z05/30 Write block 432 len 4096 data: 2
26802024-02-13T21:40:23.449Z06/30 Write block 440 len 28672 data: 3 4 4 4 4 5 5
26812024-02-13T21:40:23.449Z07/30 Read block 463 len 8192
26822024-02-13T21:40:23.449Z08/30 Read block 401 len 4096
26832024-02-13T21:40:23.449Z09/30 Read block 206 len 40960
26842024-02-13T21:40:23.454Z10/30 Write block 47 len 12288 data: 2 2 2
26852024-02-13T21:40:23.454Z11/30 Write block 256 len 40960 data: 3 3 3 2 2 2 2 2 4 4
26862024-02-13T21:40:23.454Z12/30 Read block 175 len 8192
26872024-02-13T21:40:23.454Z13/30 Read block 81 len 36864
26882024-02-13T21:40:23.458Z14/30 Read block 427 len 40960
26892024-02-13T21:40:23.459Z15/30 Read block 406 len 8192
26902024-02-13T21:40:23.459Z16/30 Read block 419 len 4096
26912024-02-13T21:40:23.459Z17/30 Write block 337 len 8192 data: 4 4
26922024-02-13T21:40:23.459Z18/30 Write block 234 len 40960 data: 4 4 4 4 5 4 4 4 4 4
26932024-02-13T21:40:23.459Z19/30 Flush
26942024-02-13T21:40:23.463Z20/30 Write block 457 len 36864 data: 2 2 2 2 2 2 2 2 2
26952024-02-13T21:40:23.463Z21/30 Read block 164 len 32768
26962024-02-13T21:40:23.467Z22/30 Write block 331 len 12288 data: 4 4 4
26972024-02-13T21:40:23.467Z23/30 Read block 251 len 12288
26982024-02-13T21:40:23.467Z24/30 Read block 140 len 32768
26992024-02-13T21:40:23.467Z25/30 Read block 15 len 16384
27002024-02-13T21:40:23.467Z26/30 Write block 565 len 32768 data: 5 5 5 6 4 4 3 3
27012024-02-13T21:40:23.472Z27/30 Write block 512 len 40960 data: 4 4 4 3 3 4 4 4 5 5
27022024-02-13T21:40:23.472Z28/30 Read block 469 len 20480
27032024-02-13T21:40:23.472Z29/30 Write block 227 len 20480 data: 4 4 3 4 4
27042024-02-13T21:40:23.472Z30/30 Read block 347 len 36864
27052024-02-13T21:40:23.472Z----------------------------------------------------------------
27062024-02-13T21:40:23.472Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:29
27072024-02-13T21:40:23.472ZStates: Active Active Active
27082024-02-13T21:40:23.472ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27092024-02-13T21:40:23.472Z 2 Acked 1001 Flush 0 Sent Done Done false
27102024-02-13T21:40:23.472Z 3 Acked 1002 Write 1 Sent Done Done false
27112024-02-13T21:40:23.473Z 4 Acked 1003 Flush 0 Sent Done Done false
27122024-02-13T21:40:23.473Z 5 Acked 1004 Write 1 Sent Done Done false
27132024-02-13T21:40:23.473Z 6 Acked 1005 Write 7 Sent Done Done false
27142024-02-13T21:40:23.473Z 7 Acked 1006 Read 2 Sent Done Done false
27152024-02-13T21:40:23.473Z 8 Acked 1007 Read 1 Sent Done Done false
27162024-02-13T21:40:23.473Z 9 Acked 1008 Read 10 Sent Done Done false
27172024-02-13T21:40:23.473Z 10 Acked 1009 Write 3 Sent Done Done false
27182024-02-13T21:40:23.473Z 11 Acked 1010 Write 10 Sent Done Done false
27192024-02-13T21:40:23.473Z 12 Acked 1011 Read 2 Sent Done Done false
27202024-02-13T21:40:23.473Z 13 Acked 1012 Read 9 Sent Done Done false
27212024-02-13T21:40:23.474Z 14 Acked 1013 Read 10 Sent Done Done false
27222024-02-13T21:40:23.474Z 15 Acked 1014 Read 2 Sent Done Done false
27232024-02-13T21:40:23.474Z 16 Acked 1015 Read 1 Sent Done Done false
27242024-02-13T21:40:23.474Z 17 Acked 1016 Write 2 Sent Done Done false
27252024-02-13T21:40:23.474Z 18 Acked 1017 Write 10 Sent Done Done false
27262024-02-13T21:40:23.474Z 19 Acked 1018 Flush 0 Sent Done Done false
27272024-02-13T21:40:23.474Z 20 Acked 1019 Write 9 Sent Done Done false
27282024-02-13T21:40:23.474Z 21 Acked 1020 Read 8 Sent Done Done false
27292024-02-13T21:40:23.474Z 22 Acked 1021 Write 3 Sent Done Done false
27302024-02-13T21:40:23.474Z 23 Acked 1022 Read 3 Sent Done Done false
27312024-02-13T21:40:23.474Z 24 Acked 1023 Read 8 Sent Done Done false
27322024-02-13T21:40:23.475Z 25 Acked 1024 Read 4 Sent Done Done false
27332024-02-13T21:40:23.475Z 26 Acked 1025 Write 8 Sent Done Done false
27342024-02-13T21:40:23.475Z 27 Acked 1026 Write 10 Sent Done Done false
27352024-02-13T21:40:23.475Z 28 Acked 1027 Read 5 Sent Done Done false
27362024-02-13T21:40:23.475Z 29 Acked 1028 Write 5 Sent Done Done false
27372024-02-13T21:40:23.475Z 30 Acked 1029 Read 9 Sent Done Done false
27382024-02-13T21:40:23.475Z STATES DS:0 DS:1 DS:2 TOTAL
27392024-02-13T21:40:23.475Z New 0 0 0 0
27402024-02-13T21:40:23.475Z Sent 29 0 0 29
27412024-02-13T21:40:23.475Z Done 0 29 29 58
27422024-02-13T21:40:23.476Z Skipped 0 0 0 0
27432024-02-13T21:40:23.476Z Error 0 0 0 0
27442024-02-13T21:40:23.476ZLast Flush: 1000 1018 1018
27452024-02-13T21:40:23.476ZDownstairs last five completed: 1000
27462024-02-13T21:40:23.476ZUpstairs last five completed: 30 29 28 27 26
27472024-02-13T21:40:23.480Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:22.300877505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1231}
27482024-02-13T21:40:23.480ZWrote out file "/tmp/test_repair_verify.data"
27492024-02-13T21:40:23.480Z
27502024-02-13T21:40:23.480ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27512024-02-13T21:40:23.496ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27522024-02-13T21:40:23.496Z 0 000-019 21 21 21 37 37 37 F F F
27532024-02-13T21:40:23.497Z 1 020-039 22 22 22 39 39 39 F F F
27542024-02-13T21:40:23.497Z 2 040-059 21 24 24 37 44 44 F F F <---
27552024-02-13T21:40:23.497Z 3 060-079 20 20 20 36 36 36 F F F
27562024-02-13T21:40:23.497Z 4 080-099 20 20 20 36 36 36 F F F
27572024-02-13T21:40:23.497Z 5 100-119 20 20 20 36 36 36 F F F
27582024-02-13T21:40:23.497Z 6 120-139 18 18 18 33 33 33 F F F
27592024-02-13T21:40:23.497Z 7 140-159 17 17 17 32 32 32 F F F
27602024-02-13T21:40:23.497Z 8 160-179 15 15 15 29 29 29 F F F
27612024-02-13T21:40:23.498Z 9 180-199 23 23 23 40 40 40 F F F
27622024-02-13T21:40:23.498Z 10 200-219 23 23 23 40 40 40 F F F
27632024-02-13T21:40:23.498Z 11 220-239 23 24 24 40 44 44 F T T <---
27642024-02-13T21:40:23.498Z 12 240-259 23 24 24 40 44 44 F F F <---
27652024-02-13T21:40:23.498Z 13 260-279 18 24 24 33 44 44 F F F <---
27662024-02-13T21:40:23.498Z 14 280-299 22 22 22 39 39 39 F F F
27672024-02-13T21:40:23.503Z 15 300-319 22 22 22 39 39 39 F F F
27682024-02-13T21:40:23.503Z 16 320-339 22 24 24 39 44 44 F T T <---
27692024-02-13T21:40:23.504Z 17 340-359 17 17 17 32 32 32 F F F
27702024-02-13T21:40:23.504Z 18 360-379 22 22 22 38 38 38 F F F
27712024-02-13T21:40:23.504Z 19 380-399 22 22 22 38 38 38 F F F
27722024-02-13T21:40:23.505Z 20 400-419 22 22 22 39 39 39 F F F
27732024-02-13T21:40:23.505Z 21 420-439 22 24 24 39 44 44 F F F <---
27742024-02-13T21:40:23.505Z 22 440-459 22 24 24 39 44 44 F T T <---
27752024-02-13T21:40:23.505Z 23 460-479 19 19 19 35 35 35 F T T <---
27762024-02-13T21:40:23.505Z 24 480-499 23 23 23 40 40 40 F F F
27772024-02-13T21:40:23.506Z 25 500-519 21 21 21 37 37 37 F T T <---
27782024-02-13T21:40:23.506Z 26 520-539 19 19 19 35 35 35 F T T <---
27792024-02-13T21:40:23.506Z 27 540-559 21 21 21 37 37 37 F F F
27802024-02-13T21:40:23.506Z 28 560-579 23 23 23 40 40 40 F T T <---
27812024-02-13T21:40:23.506Z 29 580-599 22 24 24 39 43 43 F F F <---
27822024-02-13T21:40:23.506ZMax gen: 24, Max flush: 44
27832024-02-13T21:40:23.507ZError: Difference in extent metadata found!
27842024-02-13T21:40:23.507ZA Difference in extent metadata is expected here
27852024-02-13T21:40:23.507ZOn loop 12
27862024-02-13T21:40:23.508Z
27872024-02-13T21:40:23.508ZVerifying data now
27882024-02-13T21:40:28.096ZNo change after verify
27892024-02-13T21:40:28.096ZLoop: 12 Downstairs dump after verify (and repair):
27902024-02-13T21:40:28.108ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27912024-02-13T21:40:28.108Z 0 000-019 21 21 21 37 37 37 F F F
27922024-02-13T21:40:28.108Z 1 020-039 22 22 22 39 39 39 F F F
27932024-02-13T21:40:28.108Z 2 040-059 24 24 24 44 44 44 F F F
27942024-02-13T21:40:28.108Z 3 060-079 20 20 20 36 36 36 F F F
27952024-02-13T21:40:28.108Z 4 080-099 20 20 20 36 36 36 F F F
27962024-02-13T21:40:28.108Z 5 100-119 20 20 20 36 36 36 F F F
27972024-02-13T21:40:28.114Z 6 120-139 18 18 18 33 33 33 F F F
27982024-02-13T21:40:28.114Z 7 140-159 17 17 17 32 32 32 F F F
27992024-02-13T21:40:28.114Z 8 160-179 15 15 15 29 29 29 F F F
28002024-02-13T21:40:28.114Z 9 180-199 23 23 23 40 40 40 F F F
28012024-02-13T21:40:28.114Z 10 200-219 23 23 23 40 40 40 F F F
28022024-02-13T21:40:28.114Z 11 220-239 25 25 25 45 45 45 F F F
28032024-02-13T21:40:28.114Z 12 240-259 24 24 24 44 44 44 F F F
28042024-02-13T21:40:28.114Z 13 260-279 24 24 24 44 44 44 F F F
28052024-02-13T21:40:28.115Z 14 280-299 22 22 22 39 39 39 F F F
28062024-02-13T21:40:28.115Z 15 300-319 22 22 22 39 39 39 F F F
28072024-02-13T21:40:28.115Z 16 320-339 25 25 25 45 45 45 F F F
28082024-02-13T21:40:28.115Z 17 340-359 17 17 17 32 32 32 F F F
28092024-02-13T21:40:28.115Z 18 360-379 22 22 22 38 38 38 F F F
28102024-02-13T21:40:28.115Z 19 380-399 22 22 22 38 38 38 F F F
28112024-02-13T21:40:28.115Z 20 400-419 22 22 22 39 39 39 F F F
28122024-02-13T21:40:28.115Z 21 420-439 24 24 24 44 44 44 F F F
28132024-02-13T21:40:28.115Z 22 440-459 25 25 25 45 45 45 F F F
28142024-02-13T21:40:28.115Z 23 460-479 25 25 25 45 45 45 F F F
28152024-02-13T21:40:28.115Z 24 480-499 23 23 23 40 40 40 F F F
28162024-02-13T21:40:28.115Z 25 500-519 25 25 25 45 45 45 F F F
28172024-02-13T21:40:28.115Z 26 520-539 25 25 25 45 45 45 F F F
28182024-02-13T21:40:28.115Z 27 540-559 21 21 21 37 37 37 F F F
28192024-02-13T21:40:28.115Z 28 560-579 25 25 25 45 45 45 F F F
28202024-02-13T21:40:28.116Z 29 580-599 24 24 24 43 43 43 F F F
28212024-02-13T21:40:28.116ZMax gen: 25, Max flush: 45
28222024-02-13T21:40:28.116Z
28232024-02-13T21:40:28.116ZBegin loop 13
28242024-02-13T21:40:28.116ZDownstairs to restart: 0
28252024-02-13T21:40:28.130Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.957642139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
28262024-02-13T21:40:28.130Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.957883452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
28272024-02-13T21:40:28.136Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.957923757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
28282024-02-13T21:40:28.136ZCrucible runtime is spawned
28292024-02-13T21:40:28.136Z{"msg":"Crucible d1ae3163-fa6c-49fd-a108-3464b4f3571f has session id: 0c03ece8-a0d6-42da-814a-4edfbf64bad1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.957974229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28302024-02-13T21:40:28.136Z{"msg":"Upstairs opts: Upstairs UUID: d1ae3163-fa6c-49fd-a108-3464b4f3571f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958021782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28312024-02-13T21:40:28.136Z{"msg":"Crucible stats registered with UUID: d1ae3163-fa6c-49fd-a108-3464b4f3571f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958051071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28322024-02-13T21:40:28.136Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958091646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
28332024-02-13T21:40:28.137Z{"msg":"d1ae3163-fa6c-49fd-a108-3464b4f3571f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958275989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28342024-02-13T21:40:28.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958311467Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28352024-02-13T21:40:28.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958328321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28362024-02-13T21:40:28.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958346854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28372024-02-13T21:40:28.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.95838482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28382024-02-13T21:40:28.137Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.95841108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28392024-02-13T21:40:28.137Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958435322Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28402024-02-13T21:40:28.137Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958449976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28412024-02-13T21:40:28.137Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958491091Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28422024-02-13T21:40:28.137Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958541303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28432024-02-13T21:40:28.137Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.958642616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28442024-02-13T21:40:28.137Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.9586869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28452024-02-13T21:40:28.137Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.95871467Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28462024-02-13T21:40:28.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.95934973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28472024-02-13T21:40:28.138Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.959386057Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28482024-02-13T21:40:28.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.95954229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28492024-02-13T21:40:28.138Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.959585374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28502024-02-13T21:40:28.138Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.959904949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28512024-02-13T21:40:28.138Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960012959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28522024-02-13T21:40:28.138Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960247105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28532024-02-13T21:40:28.138Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960316009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28542024-02-13T21:40:28.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960435006Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28552024-02-13T21:40:28.138Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960471533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28562024-02-13T21:40:28.138Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960490916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28572024-02-13T21:40:28.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960551354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28582024-02-13T21:40:28.138Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960587991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28592024-02-13T21:40:28.139Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960603815Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28602024-02-13T21:40:28.139Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.960813369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28612024-02-13T21:40:28.139Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.96125065Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28622024-02-13T21:40:28.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961288237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28632024-02-13T21:40:28.139Z{"msg":"[0]R flush_numbers[0..12]: [37, 39, 44, 36, 36, 36, 33, 32, 29, 40, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961305051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28642024-02-13T21:40:28.139Z{"msg":"[0]R generation[0..12]: [21, 22, 24, 20, 20, 20, 18, 17, 15, 23, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961319636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28652024-02-13T21:40:28.139Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961341827Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28662024-02-13T21:40:28.139Z{"msg":"[1]R flush_numbers[0..12]: [37, 39, 44, 36, 36, 36, 33, 32, 29, 40, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961356022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28672024-02-13T21:40:28.139Z{"msg":"[1]R generation[0..12]: [21, 22, 24, 20, 20, 20, 18, 17, 15, 23, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961369967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28682024-02-13T21:40:28.139Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961384772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28692024-02-13T21:40:28.139Z{"msg":"[2]R flush_numbers[0..12]: [37, 39, 44, 36, 36, 36, 33, 32, 29, 40, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961398667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28702024-02-13T21:40:28.139Z{"msg":"[2]R generation[0..12]: [21, 22, 24, 20, 20, 20, 18, 17, 15, 23, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961419609Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28712024-02-13T21:40:28.139Z{"msg":"Wait for a query_work_queue command to finish before sending IO
28722024-02-13T21:40:28.140Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961435613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28732024-02-13T21:40:28.140Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961449628Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28742024-02-13T21:40:28.140Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961469901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28752024-02-13T21:40:28.140Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961483156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28762024-02-13T21:40:28.140Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961496281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28772024-02-13T21:40:28.140Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961516584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28782024-02-13T21:40:28.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.96152886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28792024-02-13T21:40:28.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961546903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28802024-02-13T21:40:28.145Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961573513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28812024-02-13T21:40:28.145Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961587708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28822024-02-13T21:40:28.145Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961600294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28832024-02-13T21:40:28.145Z{"msg":"d1ae3163-fa6c-49fd-a108-3464b4f3571f is now active with session: 0c03ece8-a0d6-42da-814a-4edfbf64bad1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961613319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28842024-02-13T21:40:28.145Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.961625135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239,"session_id":"0c03ece8-a0d6-42da-814a-4edfbf64bad1"}
28852024-02-13T21:40:28.145Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:26.96163714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
28862024-02-13T21:40:28.145ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28872024-02-13T21:40:28.145ZLoading write count information from file "/tmp/test_repair_verify.data"
28882024-02-13T21:40:28.145ZRun Repair workload
28892024-02-13T21:40:28.146Z01/30 Write block 76 len 32768 data: 3 2 2 2 2 2 2 2
28902024-02-13T21:40:28.146Z02/30 Write block 307 len 4096 data: 5
28912024-02-13T21:40:28.146Z03/30 Write block 488 len 8192 data: 3 3
28922024-02-13T21:40:28.146Z04/30 Write block 418 len 36864 data: 5 5 4 4 3 3 3 3 3
28932024-02-13T21:40:28.146Z05/30 Write block 549 len 32768 data: 5 5 6 4 4 3 3 3
28942024-02-13T21:40:28.146Z06/30 Write block 469 len 28672 data: 3 3 4 4 4 4 5
28952024-02-13T21:40:28.146Z07/30 Write block 184 len 12288 data: 5 6 5
28962024-02-13T21:40:28.146Z08/30 Write block 141 len 36864 data: 2 3 3 3 3 3 3 3 3
28972024-02-13T21:40:28.146Z09/30 Write block 79 len 28672 data: 3 3 3 3 3 2 2
28982024-02-13T21:40:28.146Z10/30 Write block 570 len 4096 data: 5
28992024-02-13T21:40:28.146Z11/30 Flush
29002024-02-13T21:40:28.150Z12/30 Write block 166 len 4096 data: 2
29012024-02-13T21:40:28.150Z13/30 Read block 303 len 32768
29022024-02-13T21:40:28.150Z14/30 Write block 488 len 16384 data: 4 4 2 3
29032024-02-13T21:40:28.150Z15/30 Write block 188 len 32768 data: 5 5 5 5 5 5 5 4
29042024-02-13T21:40:28.150Z16/30 Read block 291 len 16384
29052024-02-13T21:40:28.154Z17/30 Read block 228 len 24576
29062024-02-13T21:40:28.154Z18/30 Read block 14 len 28672
29072024-02-13T21:40:28.154Z19/30 Write block 196 len 16384 data: 4 4 5 5
29082024-02-13T21:40:28.154Z20/30 Read block 593 len 8192
29092024-02-13T21:40:28.159Z21/30 Write block 545 len 12288 data: 4 4 3
29102024-02-13T21:40:28.159Z22/30 Read block 592 len 8192
29112024-02-13T21:40:28.160Z23/30 Read block 592 len 28672
29122024-02-13T21:40:28.160Z24/30 Read block 121 len 20480
29132024-02-13T21:40:28.165Z25/30 Write block 510 len 24576 data: 4 4 5 5 5 4
29142024-02-13T21:40:28.165Z26/30 Read block 39 len 16384
29152024-02-13T21:40:28.166Z27/30 Read block 27 len 4096
29162024-02-13T21:40:28.166Z28/30 Write block 291 len 12288 data: 3 3 3
29172024-02-13T21:40:28.166Z29/30 Read block 368 len 36864
29182024-02-13T21:40:28.166Z30/30 Read block 458 len 4096
29192024-02-13T21:40:28.166Z----------------------------------------------------------------
29202024-02-13T21:40:28.166Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29212024-02-13T21:40:28.166ZStates: Active Active Active
29222024-02-13T21:40:28.166ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29232024-02-13T21:40:28.166Z 1 Acked 1000 Write 8 Done Done Done false
29242024-02-13T21:40:28.166Z 2 Acked 1001 Write 1 Done Done Done false
29252024-02-13T21:40:28.166Z 3 Acked 1002 Write 2 Sent Done Done false
29262024-02-13T21:40:28.166Z 4 Acked 1003 Write 9 Sent Done Done false
29272024-02-13T21:40:28.166Z 5 Acked 1004 Write 8 Sent Done Done false
29282024-02-13T21:40:28.166Z 6 Acked 1005 Write 7 Sent Done Done false
29292024-02-13T21:40:28.167Z 7 Acked 1006 Write 3 Sent Done Done false
29302024-02-13T21:40:28.167Z 8 Acked 1007 Write 9 Sent Done Done false
29312024-02-13T21:40:28.167Z 9 Acked 1008 Write 7 Sent Done Done false
29322024-02-13T21:40:28.167Z 10 Acked 1009 Write 1 Sent Done Done false
29332024-02-13T21:40:28.167Z 11 Acked 1010 Flush 0 Sent Done Done false
29342024-02-13T21:40:28.167Z 12 Acked 1011 Write 1 Sent Done Done false
29352024-02-13T21:40:28.167Z 13 Acked 1012 Read 8 Sent Done Done false
29362024-02-13T21:40:28.167Z 14 Acked 1013 Write 4 Sent Done Done false
29372024-02-13T21:40:28.167Z 15 Acked 1014 Write 8 Sent Done Done false
29382024-02-13T21:40:28.167Z 16 Acked 1015 Read 4 Sent Done Done false
29392024-02-13T21:40:28.167Z 17 Acked 1016 Read 6 Sent Done Done false
29402024-02-13T21:40:28.167Z 18 Acked 1017 Read 7 Sent Done Done false
29412024-02-13T21:40:28.167Z 19 Acked 1018 Write 4 Sent Done Done false
29422024-02-13T21:40:28.167Z 20 Acked 1019 Read 2 Sent Done Done false
29432024-02-13T21:40:28.167Z 21 Acked 1020 Write 3 Sent Done Done false
29442024-02-13T21:40:28.167Z 22 Acked 1021 Read 2 Sent Done Done false
29452024-02-13T21:40:28.167Z 23 Acked 1022 Read 7 Sent Done Done false
29462024-02-13T21:40:28.167Z 24 Acked 1023 Read 5 Sent Done Done false
29472024-02-13T21:40:28.167Z 25 Acked 1024 Write 6 Sent Done Done false
29482024-02-13T21:40:28.167Z 26 Acked 1025 Read 4 Sent Done Done false
29492024-02-13T21:40:28.167Z 27 Acked 1026 Read 1 Sent Done Done false
29502024-02-13T21:40:28.168Z 28 Acked 1027 Write 3 Sent Done Done false
29512024-02-13T21:40:28.168Z 29 Acked 1028 Read 9 Sent Done Done false
29522024-02-13T21:40:28.168Z 30 Acked 1029 Read 1 Sent Done Done false
29532024-02-13T21:40:28.168Z STATES DS:0 DS:1 DS:2 TOTAL
29542024-02-13T21:40:28.168Z New 0 0 0 0
29552024-02-13T21:40:28.168Z Sent 28 0 0 28
29562024-02-13T21:40:28.168Z Done 2 30 30 62
29572024-02-13T21:40:28.168Z Skipped 0 0 0 0
29582024-02-13T21:40:28.168Z Error 0 0 0 0
29592024-02-13T21:40:28.168ZLast Flush: 0 1010 1010
29602024-02-13T21:40:28.168ZDownstairs last five completed:
29612024-02-13T21:40:28.168ZUpstairs last five completed: 30 29 28 27 26
29622024-02-13T21:40:28.168Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:26.990237386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1239}
29632024-02-13T21:40:28.168ZWrote out file "/tmp/test_repair_verify.data"
29642024-02-13T21:40:28.171Z
29652024-02-13T21:40:28.171ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29662024-02-13T21:40:28.187ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29672024-02-13T21:40:28.187Z 0 000-019 21 21 21 37 37 37 F F F
29682024-02-13T21:40:28.187Z 1 020-039 22 22 22 39 39 39 F F F
29692024-02-13T21:40:28.187Z 2 040-059 24 24 24 44 44 44 F F F
29702024-02-13T21:40:28.187Z 3 060-079 20 26 26 36 46 46 T F F <---
29712024-02-13T21:40:28.187Z 4 080-099 20 26 26 36 46 46 T F F <---
29722024-02-13T21:40:28.188Z 5 100-119 20 20 20 36 36 36 F F F
29732024-02-13T21:40:28.188Z 6 120-139 18 18 18 33 33 33 F F F
29742024-02-13T21:40:28.188Z 7 140-159 17 26 26 32 46 46 F F F <---
29752024-02-13T21:40:28.188Z 8 160-179 15 15 15 29 29 29 F T T <---
29762024-02-13T21:40:28.188Z 9 180-199 23 26 26 40 46 46 F T T <---
29772024-02-13T21:40:28.188Z 10 200-219 23 23 23 40 40 40 F F F
29782024-02-13T21:40:28.191Z 11 220-239 25 25 25 45 45 45 F F F
29792024-02-13T21:40:28.191Z 12 240-259 24 24 24 44 44 44 F F F
29802024-02-13T21:40:28.192Z 13 260-279 24 24 24 44 44 44 F F F
29812024-02-13T21:40:28.192Z 14 280-299 22 22 22 39 39 39 F T T <---
29822024-02-13T21:40:28.192Z 15 300-319 22 26 26 39 46 46 T F F <---
29832024-02-13T21:40:28.192Z 16 320-339 25 25 25 45 45 45 F F F
29842024-02-13T21:40:28.192Z 17 340-359 17 17 17 32 32 32 F F F
29852024-02-13T21:40:28.192Z 18 360-379 22 22 22 38 38 38 F F F
29862024-02-13T21:40:28.192Z 19 380-399 22 22 22 38 38 38 F F F
29872024-02-13T21:40:28.192Z 20 400-419 22 26 26 39 46 46 F F F <---
29882024-02-13T21:40:28.192Z 21 420-439 24 26 26 44 46 46 F F F <---
29892024-02-13T21:40:28.192Z 22 440-459 25 25 25 45 45 45 F F F
29902024-02-13T21:40:28.192Z 23 460-479 25 26 26 45 46 46 F F F <---
29912024-02-13T21:40:28.192Z 24 480-499 23 26 26 40 46 46 F T T <---
29922024-02-13T21:40:28.193Z 25 500-519 25 25 25 45 45 45 F T T <---
29932024-02-13T21:40:28.193Z 26 520-539 25 25 25 45 45 45 F F F
29942024-02-13T21:40:28.193Z 27 540-559 21 26 26 37 46 46 F T T <---
29952024-02-13T21:40:28.193Z 28 560-579 25 26 26 45 46 46 F F F <---
29962024-02-13T21:40:28.193Z 29 580-599 24 24 24 43 43 43 F F F
29972024-02-13T21:40:28.193ZMax gen: 26, Max flush: 46
29982024-02-13T21:40:28.193ZError: Difference in extent metadata found!
29992024-02-13T21:40:28.193ZA Difference in extent metadata is expected here
30002024-02-13T21:40:28.193ZOn loop 13
30012024-02-13T21:40:28.193Z
30022024-02-13T21:40:28.197ZVerifying data now
30032024-02-13T21:40:33.868ZNo change after verify
30042024-02-13T21:40:33.868ZLoop: 13 Downstairs dump after verify (and repair):
30052024-02-13T21:40:33.883ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30062024-02-13T21:40:33.883Z 0 000-019 21 21 21 37 37 37 F F F
30072024-02-13T21:40:33.883Z 1 020-039 22 22 22 39 39 39 F F F
30082024-02-13T21:40:33.883Z 2 040-059 24 24 24 44 44 44 F F F
30092024-02-13T21:40:33.883Z 3 060-079 26 26 26 46 46 46 F F F
30102024-02-13T21:40:33.883Z 4 080-099 26 26 26 46 46 46 F F F
30112024-02-13T21:40:33.883Z 5 100-119 20 20 20 36 36 36 F F F
30122024-02-13T21:40:33.883Z 6 120-139 18 18 18 33 33 33 F F F
30132024-02-13T21:40:33.887Z 7 140-159 26 26 26 46 46 46 F F F
30142024-02-13T21:40:33.887Z 8 160-179 27 27 27 47 47 47 F F F
30152024-02-13T21:40:33.887Z 9 180-199 27 27 27 47 47 47 F F F
30162024-02-13T21:40:33.887Z 10 200-219 23 23 23 40 40 40 F F F
30172024-02-13T21:40:33.887Z 11 220-239 25 25 25 45 45 45 F F F
30182024-02-13T21:40:33.887Z 12 240-259 24 24 24 44 44 44 F F F
30192024-02-13T21:40:33.887Z 13 260-279 24 24 24 44 44 44 F F F
30202024-02-13T21:40:33.887Z 14 280-299 27 27 27 47 47 47 F F F
30212024-02-13T21:40:33.887Z 15 300-319 26 26 26 46 46 46 F F F
30222024-02-13T21:40:33.887Z 16 320-339 25 25 25 45 45 45 F F F
30232024-02-13T21:40:33.887Z 17 340-359 17 17 17 32 32 32 F F F
30242024-02-13T21:40:33.887Z 18 360-379 22 22 22 38 38 38 F F F
30252024-02-13T21:40:33.887Z 19 380-399 22 22 22 38 38 38 F F F
30262024-02-13T21:40:33.887Z 20 400-419 26 26 26 46 46 46 F F F
30272024-02-13T21:40:33.887Z 21 420-439 26 26 26 46 46 46 F F F
30282024-02-13T21:40:33.887Z 22 440-459 25 25 25 45 45 45 F F F
30292024-02-13T21:40:33.887Z 23 460-479 26 26 26 46 46 46 F F F
30302024-02-13T21:40:33.888Z 24 480-499 27 27 27 47 47 47 F F F
30312024-02-13T21:40:33.888Z 25 500-519 27 27 27 47 47 47 F F F
30322024-02-13T21:40:33.888Z 26 520-539 25 25 25 45 45 45 F F F
30332024-02-13T21:40:33.888Z 27 540-559 27 27 27 47 47 47 F F F
30342024-02-13T21:40:33.888Z 28 560-579 26 26 26 46 46 46 F F F
30352024-02-13T21:40:33.888Z 29 580-599 24 24 24 43 43 43 F F F
30362024-02-13T21:40:33.888ZMax gen: 27, Max flush: 47
30372024-02-13T21:40:33.888Z
30382024-02-13T21:40:33.888ZBegin loop 14
30392024-02-13T21:40:33.888ZDownstairs to restart: 2
30402024-02-13T21:40:33.905Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.730645892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
30412024-02-13T21:40:33.905Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.730879487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
30422024-02-13T21:40:33.905Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.730903328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
30432024-02-13T21:40:33.905ZCrucible runtime is spawned
30442024-02-13T21:40:33.935Z{"msg":"Crucible aeb811e8-8088-4606-b3f4-75cac2130398 has session id: a14e06c4-3847-417d-875a-ca7beae17e78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.730965896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30452024-02-13T21:40:33.935Z{"msg":"Upstairs opts: Upstairs UUID: aeb811e8-8088-4606-b3f4-75cac2130398, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731020426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30462024-02-13T21:40:33.935Z{"msg":"Crucible stats registered with UUID: aeb811e8-8088-4606-b3f4-75cac2130398","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731058512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30472024-02-13T21:40:33.935Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731084453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
30482024-02-13T21:40:33.935Z{"msg":"aeb811e8-8088-4606-b3f4-75cac2130398 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731168272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30492024-02-13T21:40:33.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731196012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30502024-02-13T21:40:33.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731211667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30512024-02-13T21:40:33.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731242915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30522024-02-13T21:40:33.936Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731261529Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30532024-02-13T21:40:33.936Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731297126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30542024-02-13T21:40:33.936Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73131244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30552024-02-13T21:40:33.936Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731325985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30562024-02-13T21:40:33.936Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731373358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30572024-02-13T21:40:33.936Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731391102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30582024-02-13T21:40:33.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731426679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30592024-02-13T21:40:33.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731482238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30602024-02-13T21:40:33.936Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.731521104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30612024-02-13T21:40:33.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732233866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30622024-02-13T21:40:33.936Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732269113Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30632024-02-13T21:40:33.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732284678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30642024-02-13T21:40:33.936Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732314696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30652024-02-13T21:40:33.936Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732860689Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30662024-02-13T21:40:33.936Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.732895436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30672024-02-13T21:40:33.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73337951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30682024-02-13T21:40:33.936Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733415387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30692024-02-13T21:40:33.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733430852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30702024-02-13T21:40:33.937Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733538812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30712024-02-13T21:40:33.937Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733570271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30722024-02-13T21:40:33.937Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733585386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30732024-02-13T21:40:33.937Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733597881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30742024-02-13T21:40:33.937Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.733619263Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30752024-02-13T21:40:33.937Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734077067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30762024-02-13T21:40:33.937Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734471864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30772024-02-13T21:40:33.937Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73451111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30782024-02-13T21:40:33.937Z{"msg":"[0]R flush_numbers[0..12]: [37, 39, 44, 46, 46, 36, 33, 46, 47, 47, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734558163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30792024-02-13T21:40:33.937Z{"msg":"[0]R generation[0..12]: [21, 22, 24, 26, 26, 20, 18, 26, 27, 27, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73459441Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30802024-02-13T21:40:33.937Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734611363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30812024-02-13T21:40:33.937Z{"msg":"[1]R flush_numbers[0..12]: [37, 39, 44, 46, 46, 36, 33, 46, 47, 47, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734633036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30822024-02-13T21:40:33.937Z{"msg":"[1]R generation[0..12]: [21, 22, 24, 26, 26, 20, 18, 26, 27, 27, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73464781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30832024-02-13T21:40:33.937ZWait for a query_work_queue command to finish before sending IO
30842024-02-13T21:40:33.937Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.7346766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30852024-02-13T21:40:33.937Z{"msg":"[2]R flush_numbers[0..12]: [37, 39, 44, 46, 46, 36, 33, 46, 47, 47, 40, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734693424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30862024-02-13T21:40:33.937Z{"msg":"[2]R generation[0..12]: [21, 22, 24, 26, 26, 20, 18, 26, 27, 27, 23, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734715436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30872024-02-13T21:40:33.937Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734729971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30882024-02-13T21:40:33.937Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734749773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30892024-02-13T21:40:33.937Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734764738Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30902024-02-13T21:40:33.937Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734778243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30912024-02-13T21:40:33.938Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734793168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30922024-02-13T21:40:33.938Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734824946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30932024-02-13T21:40:33.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73484224Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30942024-02-13T21:40:33.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734877807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30952024-02-13T21:40:33.938Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734893071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30962024-02-13T21:40:33.938Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734929068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30972024-02-13T21:40:33.938Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734949311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30982024-02-13T21:40:33.938Z{"msg":"aeb811e8-8088-4606-b3f4-75cac2130398 is now active with session: a14e06c4-3847-417d-875a-ca7beae17e78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.73497913Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
30992024-02-13T21:40:33.938Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.734993765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247,"session_id":"a14e06c4-3847-417d-875a-ca7beae17e78"}
31002024-02-13T21:40:33.938Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:32.735052274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
31012024-02-13T21:40:33.938ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31022024-02-13T21:40:33.938ZLoading write count information from file "/tmp/test_repair_verify.data"
31032024-02-13T21:40:33.938ZRun Repair workload
31042024-02-13T21:40:33.938Z01/30 Write block 293 len 36864 data: 4 3 3 3 4 3 3 3 2
31052024-02-13T21:40:33.938Z02/30 Write block 334 len 12288 data: 4 5 5
31062024-02-13T21:40:33.938Z03/30 Write block 513 len 12288 data: 6 6 5
31072024-02-13T21:40:33.939Z04/30 Read block 342 len 28672
31082024-02-13T21:40:33.939Z05/30 Read block 231 len 32768
31092024-02-13T21:40:33.939Z06/30 Read block 255 len 16384
31102024-02-13T21:40:33.939Z07/30 Write block 9 len 4096 data: 2
31112024-02-13T21:40:33.939Z08/30 Write block 44 len 40960 data: 2 2 2 3 3 3 2 2 2 2
31122024-02-13T21:40:33.939Z09/30 Write block 157 len 28672 data: 3 3 2 2 2 2 2
31132024-02-13T21:40:33.939Z10/30 Write block 514 len 4096 data: 7
31142024-02-13T21:40:33.939Z11/30 Flush
31152024-02-13T21:40:33.939Z12/30 Write block 97 len 32768 data: 3 2 2 2 2 3 3 2
31162024-02-13T21:40:33.939Z13/30 Write block 120 len 36864 data: 2 3 3 3 4 5 6 6 4
31172024-02-13T21:40:33.939Z14/30 Write block 428 len 16384 data: 3 3 3 2
31182024-02-13T21:40:33.939Z15/30 Write block 210 len 28672 data: 4 4 4 5 4 4 6
31192024-02-13T21:40:33.939Z16/30 Write block 432 len 20480 data: 3 2 3 3 2
31202024-02-13T21:40:33.939Z17/30 Read block 367 len 32768
31212024-02-13T21:40:33.939Z18/30 Write block 510 len 40960 data: 5 5 6 7 8 6 4 5 5 5
31222024-02-13T21:40:33.939Z19/30 Write block 172 len 12288 data: 2 2 2
31232024-02-13T21:40:33.940Z20/30 Read block 284 len 12288
31242024-02-13T21:40:33.940Z21/30 Write block 117 len 40960 data: 2 2 2 3 4 4 4 5 6 7
31252024-02-13T21:40:33.940Z22/30 Write block 135 len 16384 data: 2 2 2 2
31262024-02-13T21:40:33.940Z23/30 Read block 76 len 28672
31272024-02-13T21:40:33.945Z24/30 Read block 350 len 12288
31282024-02-13T21:40:33.946Z25/30 Read block 245 len 32768
31292024-02-13T21:40:33.946Z26/30 Read block 504 len 40960
31302024-02-13T21:40:33.946Z27/30 Write block 347 len 4096 data: 4
31312024-02-13T21:40:33.946Z28/30 Read block 444 len 24576
31322024-02-13T21:40:33.946Z29/30 Write block 538 len 32768 data: 3 3 3 4 3 3 4 5
31332024-02-13T21:40:33.946Z30/30 Read block 284 len 40960
31342024-02-13T21:40:33.946Z----------------------------------------------------------------
31352024-02-13T21:40:33.946Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31362024-02-13T21:40:33.946ZStates: Active Active Active
31372024-02-13T21:40:33.946ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31382024-02-13T21:40:33.946Z 1 Acked 1000 Write 9 Done Done Done false
31392024-02-13T21:40:33.946Z 2 Acked 1001 Write 3 Done Done Done false
31402024-02-13T21:40:33.946Z 3 Acked 1002 Write 3 Done Done Done false
31412024-02-13T21:40:33.947Z 4 Acked 1003 Read 7 Done Done Sent false
31422024-02-13T21:40:33.947Z 5 Acked 1004 Read 8 Done Done Sent false
31432024-02-13T21:40:33.947Z 6 Acked 1005 Read 4 Done Done Sent false
31442024-02-13T21:40:33.947Z 7 Acked 1006 Write 1 Done Done Sent false
31452024-02-13T21:40:33.947Z 8 Acked 1007 Write 10 Done Done Sent false
31462024-02-13T21:40:33.947Z 9 Acked 1008 Write 7 Done Done Sent false
31472024-02-13T21:40:33.947Z 10 Acked 1009 Write 1 Done Done Sent false
31482024-02-13T21:40:33.947Z 11 Acked 1010 Flush 0 Done Done Sent false
31492024-02-13T21:40:33.947Z 12 Acked 1011 Write 8 Done Done Sent false
31502024-02-13T21:40:33.947Z 13 Acked 1012 Write 9 Done Done Sent false
31512024-02-13T21:40:33.947Z 14 Acked 1013 Write 4 Done Done Sent false
31522024-02-13T21:40:33.947Z 15 Acked 1014 Write 7 Done Done Sent false
31532024-02-13T21:40:33.947Z 16 Acked 1015 Write 5 Done Done Sent false
31542024-02-13T21:40:33.948Z 17 Acked 1016 Read 8 Done Done Sent false
31552024-02-13T21:40:33.948Z 18 Acked 1017 Write 10 Done Done Sent false
31562024-02-13T21:40:33.948Z 19 Acked 1018 Write 3 Done Done Sent false
31572024-02-13T21:40:33.948Z 20 Acked 1019 Read 3 Done Done Sent false
31582024-02-13T21:40:33.948Z 21 Acked 1020 Write 10 Done Done Sent false
31592024-02-13T21:40:33.948Z 22 Acked 1021 Write 4 Done Done Sent false
31602024-02-13T21:40:33.948Z 23 Acked 1022 Read 7 Done Done Sent false
31612024-02-13T21:40:33.948Z 24 Acked 1023 Read 3 Done Done Sent false
31622024-02-13T21:40:33.948Z 25 Acked 1024 Read 8 Done Done Sent false
31632024-02-13T21:40:33.948Z 26 Acked 1025 Read 10 Done Done Sent false
31642024-02-13T21:40:33.948Z 27 Acked 1026 Write 1 Done Done Sent false
31652024-02-13T21:40:33.949Z 28 Acked 1027 Read 6 Done Done Sent false
31662024-02-13T21:40:33.949Z 29 Acked 1028 Write 8 Done Done Sent false
31672024-02-13T21:40:33.949Z 30 Acked 1029 Read 10 Done Done Sent false
31682024-02-13T21:40:33.949Z STATES DS:0 DS:1 DS:2 TOTAL
31692024-02-13T21:40:33.949Z New 0 0 0 0
31702024-02-13T21:40:33.949Z Sent 0 0 27 27
31712024-02-13T21:40:33.949Z Done 30 30 3 63
31722024-02-13T21:40:33.949Z Skipped 0 0 0 0
31732024-02-13T21:40:33.949Z Error 0 0 0 0
31742024-02-13T21:40:33.949ZLast Flush: 1010 1010 0
31752024-02-13T21:40:33.949ZDownstairs last five completed:
31762024-02-13T21:40:33.949ZUpstairs last five completed: 30 29 28 27 26
31772024-02-13T21:40:33.949Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:32.76455992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1247}
31782024-02-13T21:40:33.949ZWrote out file "/tmp/test_repair_verify.data"
31792024-02-13T21:40:33.949Z
31802024-02-13T21:40:33.955ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31812024-02-13T21:40:33.964ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31822024-02-13T21:40:33.964Z 0 000-019 28 28 21 48 48 37 F F F <---
31832024-02-13T21:40:33.965Z 1 020-039 22 22 22 39 39 39 F F F
31842024-02-13T21:40:33.965Z 2 040-059 28 28 24 48 48 44 F F F <---
31852024-02-13T21:40:33.965Z 3 060-079 26 26 26 46 46 46 F F F
31862024-02-13T21:40:33.965Z 4 080-099 26 26 26 46 46 46 T T F <---
31872024-02-13T21:40:33.965Z 5 100-119 20 20 20 36 36 36 T T F <---
31882024-02-13T21:40:33.965Z 6 120-139 18 18 18 33 33 33 T T F <---
31892024-02-13T21:40:33.965Z 7 140-159 28 28 26 48 48 46 F F F <---
31902024-02-13T21:40:33.965Z 8 160-179 28 28 27 48 48 47 T T F <---
31912024-02-13T21:40:33.965Z 9 180-199 27 27 27 47 47 47 F F F
31922024-02-13T21:40:33.965Z 10 200-219 23 23 23 40 40 40 T T F <---
31932024-02-13T21:40:33.968Z 11 220-239 25 25 25 45 45 45 F F F
31942024-02-13T21:40:33.968Z 12 240-259 24 24 24 44 44 44 F F F
31952024-02-13T21:40:33.968Z 13 260-279 24 24 24 44 44 44 F F F
31962024-02-13T21:40:33.968Z 14 280-299 28 28 27 48 48 47 F F T <---
31972024-02-13T21:40:33.968Z 15 300-319 28 28 26 48 48 46 F F T <---
31982024-02-13T21:40:33.969Z 16 320-339 28 28 25 48 48 45 F F T <---
31992024-02-13T21:40:33.969Z 17 340-359 17 17 17 32 32 32 T T F <---
32002024-02-13T21:40:33.969Z 18 360-379 22 22 22 38 38 38 F F F
32012024-02-13T21:40:33.969Z 19 380-399 22 22 22 38 38 38 F F F
32022024-02-13T21:40:33.969Z 20 400-419 26 26 26 46 46 46 F F F
32032024-02-13T21:40:33.969Z 21 420-439 26 26 26 46 46 46 T T F <---
32042024-02-13T21:40:33.969Z 22 440-459 25 25 25 45 45 45 F F F
32052024-02-13T21:40:33.969Z 23 460-479 26 26 26 46 46 46 F F F
32062024-02-13T21:40:33.969Z 24 480-499 27 27 27 47 47 47 F F F
32072024-02-13T21:40:33.969Z 25 500-519 28 28 27 48 48 47 T T T <---
32082024-02-13T21:40:33.969Z 26 520-539 25 25 25 45 45 45 T T F <---
32092024-02-13T21:40:33.969Z 27 540-559 27 27 27 47 47 47 T T F <---
32102024-02-13T21:40:33.969Z 28 560-579 26 26 26 46 46 46 F F F
32112024-02-13T21:40:33.969Z 29 580-599 24 24 24 43 43 43 F F F
32122024-02-13T21:40:33.969ZMax gen: 28, Max flush: 48
32132024-02-13T21:40:33.969ZError: Difference in extent metadata found!
32142024-02-13T21:40:33.969ZA Difference in extent metadata is expected here
32152024-02-13T21:40:33.969ZOn loop 14
32162024-02-13T21:40:33.969Z
32172024-02-13T21:40:33.972ZVerifying data now
32182024-02-13T21:40:39.770ZNo change after verify
32192024-02-13T21:40:39.770ZLoop: 14 Downstairs dump after verify (and repair):
32202024-02-13T21:40:39.798ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32212024-02-13T21:40:39.798Z 0 000-019 28 28 28 48 48 48 F F F
32222024-02-13T21:40:39.798Z 1 020-039 22 22 22 39 39 39 F F F
32232024-02-13T21:40:39.798Z 2 040-059 28 28 28 48 48 48 F F F
32242024-02-13T21:40:39.798Z 3 060-079 26 26 26 46 46 46 F F F
32252024-02-13T21:40:39.798Z 4 080-099 29 29 29 49 49 49 F F F
32262024-02-13T21:40:39.799Z 5 100-119 29 29 29 49 49 49 F F F
32272024-02-13T21:40:39.799Z 6 120-139 29 29 29 49 49 49 F F F
32282024-02-13T21:40:39.799Z 7 140-159 28 28 28 48 48 48 F F F
32292024-02-13T21:40:39.799Z 8 160-179 29 29 29 49 49 49 F F F
32302024-02-13T21:40:39.799Z 9 180-199 27 27 27 47 47 47 F F F
32312024-02-13T21:40:39.799Z 10 200-219 29 29 29 49 49 49 F F F
32322024-02-13T21:40:39.800Z 11 220-239 25 25 25 45 45 45 F F F
32332024-02-13T21:40:39.800Z 12 240-259 24 24 24 44 44 44 F F F
32342024-02-13T21:40:39.800Z 13 260-279 24 24 24 44 44 44 F F F
32352024-02-13T21:40:39.800Z 14 280-299 28 28 28 48 48 48 F F F
32362024-02-13T21:40:39.800Z 15 300-319 28 28 28 48 48 48 F F F
32372024-02-13T21:40:39.801Z 16 320-339 28 28 28 48 48 48 F F F
32382024-02-13T21:40:39.801Z 17 340-359 29 29 29 49 49 49 F F F
32392024-02-13T21:40:39.801Z 18 360-379 22 22 22 38 38 38 F F F
32402024-02-13T21:40:39.801Z 19 380-399 22 22 22 38 38 38 F F F
32412024-02-13T21:40:39.801Z 20 400-419 26 26 26 46 46 46 F F F
32422024-02-13T21:40:39.801Z 21 420-439 29 29 29 49 49 49 F F F
32432024-02-13T21:40:39.801Z 22 440-459 25 25 25 45 45 45 F F F
32442024-02-13T21:40:39.802Z 23 460-479 26 26 26 46 46 46 F F F
32452024-02-13T21:40:39.802Z 24 480-499 27 27 27 47 47 47 F F F
32462024-02-13T21:40:39.802Z 25 500-519 29 29 29 49 49 49 F F F
32472024-02-13T21:40:39.802Z 26 520-539 29 29 29 49 49 49 F F F
32482024-02-13T21:40:39.803Z 27 540-559 29 29 29 49 49 49 F F F
32492024-02-13T21:40:39.803Z 28 560-579 26 26 26 46 46 46 F F F
32502024-02-13T21:40:39.803Z 29 580-599 24 24 24 43 43 43 F F F
32512024-02-13T21:40:39.803ZMax gen: 29, Max flush: 49
32522024-02-13T21:40:39.803Z
32532024-02-13T21:40:39.803ZBegin loop 15
32542024-02-13T21:40:39.804ZDownstairs to restart: 0
32552024-02-13T21:40:39.831Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613234415Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
32562024-02-13T21:40:39.831Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613457484Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
32572024-02-13T21:40:39.831Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613474378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
32582024-02-13T21:40:39.831Z{"msg":"Crucible 7fd479bc-2812-4313-b1db-34202723af4c has session id: 7f3ab097-23a4-443f-ab65-38ad9e728758","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613505877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32592024-02-13T21:40:39.831Z{"msg":"Upstairs opts: Upstairs UUID: 7fd479bc-2812-4313-b1db-34202723af4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
32602024-02-13T21:40:39.831Z,"time":"2024-02-13T21:40:38.6135532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32612024-02-13T21:40:39.831Z{"msg":"Crucible stats registered with UUID: 7fd479bc-2812-4313-b1db-34202723af4c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613599373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32622024-02-13T21:40:39.831Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613652344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
32632024-02-13T21:40:39.831Z{"msg":"7fd479bc-2812-4313-b1db-34202723af4c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613788424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32642024-02-13T21:40:39.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613850062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32652024-02-13T21:40:39.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.61388331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32662024-02-13T21:40:39.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613925045Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32672024-02-13T21:40:39.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.613956403Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32682024-02-13T21:40:39.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614003806Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32692024-02-13T21:40:39.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614033805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32702024-02-13T21:40:39.832Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614061385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32712024-02-13T21:40:39.832Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614113646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32722024-02-13T21:40:39.832Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614147474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32732024-02-13T21:40:39.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.61418538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32742024-02-13T21:40:39.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614210641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32752024-02-13T21:40:39.832Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614252336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32762024-02-13T21:40:39.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.6148748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32772024-02-13T21:40:39.833Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614921333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32782024-02-13T21:40:39.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614952492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32792024-02-13T21:40:39.833Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.614981062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32802024-02-13T21:40:39.833Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.615567299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32812024-02-13T21:40:39.833Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.615608964Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32822024-02-13T21:40:39.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.615944782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32832024-02-13T21:40:39.834Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.615982149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32842024-02-13T21:40:39.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616121858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32852024-02-13T21:40:39.834Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616155786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32862024-02-13T21:40:39.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.61617121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32872024-02-13T21:40:39.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616190453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32882024-02-13T21:40:39.834Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616216713Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32892024-02-13T21:40:39.835Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616230459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32902024-02-13T21:40:39.835Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616472571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32912024-02-13T21:40:39.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.61685912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32922024-02-13T21:40:39.835Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616890929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32932024-02-13T21:40:39.835Z{"msg":"[0]R flush_numbers[0..12]: [48, 39, 48, 46, 49, 49, 49, 48, 49, 47, 49, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616905574Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32942024-02-13T21:40:39.835Z{"msg":"[0]R generation[0..12]: [28, 22, 28, 26, 29, 29, 29, 28, 29, 27, 29, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616926826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32952024-02-13T21:40:39.835Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.616981536Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32962024-02-13T21:40:39.835Z{"msg":"[1]R flush_numbers[0..12]: [48, 39, 48, 46, 49, 49, 49, 48, 49, 47, 49, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617007107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32972024-02-13T21:40:39.835Z{"msg":"[1]R generation[0..12]: [28, 22, 28, 26, 29, 29, 29, 28, 29, 27, 29, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617021541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32982024-02-13T21:40:39.835Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617043993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
32992024-02-13T21:40:39.836Z{"msg":"[2]R flush_numbers[0..12]: [48, 39, 48, 46, 49, 49, 49, 48, 49, 47, 49, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617058538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33002024-02-13T21:40:39.836Z{"msg":"[2]R generation[0..12]: [28, 22, 28, 26, 29, 29, 29, 28, 29, 27, 29, 25]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
33012024-02-13T21:40:39.836Z,"time":"2024-02-13T21:40:38.617078451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33022024-02-13T21:40:39.836Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617094675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33032024-02-13T21:40:39.836Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617115487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33042024-02-13T21:40:39.836Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617129622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33052024-02-13T21:40:39.836Z{"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617142677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33062024-02-13T21:40:39.836Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617155843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33072024-02-13T21:40:39.836Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617175965Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33082024-02-13T21:40:39.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617188591Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33092024-02-13T21:40:39.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617208374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33102024-02-13T21:40:39.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617221919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33112024-02-13T21:40:39.837Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617241782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33122024-02-13T21:40:39.837Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617254677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33132024-02-13T21:40:39.837Z{"msg":"7fd479bc-2812-4313-b1db-34202723af4c is now active with session: 7f3ab097-23a4-443f-ab65-38ad9e728758","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.61727374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33142024-02-13T21:40:39.837Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617285766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255,"session_id":"7f3ab097-23a4-443f-ab65-38ad9e728758"}
33152024-02-13T21:40:39.837Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:38.617298921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
33162024-02-13T21:40:39.838ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33172024-02-13T21:40:39.838ZLoading write count information from file "/tmp/test_repair_verify.data"
33182024-02-13T21:40:39.838ZRun Repair workload
33192024-02-13T21:40:39.838Z01/30 Write block 101 len 32768 data: 3 4 4 3 2 2 2 2
33202024-02-13T21:40:39.838Z02/30 Write block 16 len 12288 data: 6 7 8
33212024-02-13T21:40:39.838Z03/30 Write block 582 len 28672 data: 2 4 3 4 4 4 3
33222024-02-13T21:40:39.838Z04/30 Write block 534 len 32768 data: 3 3 3 3 4 4 4 5
33232024-02-13T21:40:39.838Z05/30 Read block 129 len 12288
33242024-02-13T21:40:39.838Z06/30 Write block 230 len 40960 data: 5 5 4 3 5 5 5 5 6 5
33252024-02-13T21:40:39.838Z07/30 Write block 49 len 8192 data: 4 3
33262024-02-13T21:40:39.839Z08/30 Write block 469 len 24576 data: 4 4 5 5 5 5
33272024-02-13T21:40:39.839Z09/30 Read block 229 len 28672
33282024-02-13T21:40:39.839Z10/30 Write block 82 len 20480 data: 4 4 3 3 2
33292024-02-13T21:40:39.839Z11/30 Write block 273 len 32768 data: 2 2 2 2 2 2 2 2
33302024-02-13T21:40:39.839Z12/30 Read block 47 len 8192
33312024-02-13T21:40:39.839Z13/30 Read block 122 len 36864
33322024-02-13T21:40:39.839Z14/30 Read block 139 len 16384
33332024-02-13T21:40:39.839Z15/30 Read block 147 len 8192
33342024-02-13T21:40:39.840Z16/30 Write block 217 len 16384 data: 7 6 6 5
33352024-02-13T21:40:39.840Z17/30 Write block 313 len 20480 data: 2 2 2 2 2
33362024-02-13T21:40:39.861Z18/30 Write block 117 len 4096 data: 3
33372024-02-13T21:40:39.862Z19/30 Read block 101 len 4096
33382024-02-13T21:40:39.862Z20/30 Read block 488 len 8192
33392024-02-13T21:40:39.862Z21/30 Read block 404 len 36864
33402024-02-13T21:40:39.862Z22/30 Read block 591 len 24576
33412024-02-13T21:40:39.862Z23/30 Write block 447 len 28672 data: 5 5 5 4 5 4 4
33422024-02-13T21:40:39.862Z24/30 Read block 178 len 4096
33432024-02-13T21:40:39.862Z25/30 Write block 332 len 36864 data: 5 5 5 6 6 5 5 4 4
33442024-02-13T21:40:39.862Z26/30 Read block 172 len 8192
33452024-02-13T21:40:39.863Z27/30 Write block 235 len 28672 data: 6 6 6 7 6 5 5
33462024-02-13T21:40:39.863Z28/30 Read block 267 len 40960
33472024-02-13T21:40:39.863Z29/30 Read block 217 len 16384
33482024-02-13T21:40:39.863Z30/30 Write block 459 len 4096 data: 3
33492024-02-13T21:40:39.863Z----------------------------------------------------------------
33502024-02-13T21:40:39.863Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33512024-02-13T21:40:39.863ZStates: Active Active Active
33522024-02-13T21:40:39.863ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33532024-02-13T21:40:39.863Z 1 Acked 1000 Write 8 Done Done Done false
33542024-02-13T21:40:39.863Z 2 Acked 1001 Write 3 Done Done Done false
33552024-02-13T21:40:39.864Z 3 Acked 1002 Write 7 Done Done Done false
33562024-02-13T21:40:39.864Z 4 Acked 1003 Write 8 Done Done Done false
33572024-02-13T21:40:39.864Z 5 Acked 1004 Read 3 Done Done Done false
33582024-02-13T21:40:39.864Z 6 Acked 1005 Write 10 Done Done Done false
33592024-02-13T21:40:39.864Z 7 Acked 1006 Write 2 Sent Done Done false
33602024-02-13T21:40:39.864Z 8 Acked 1007 Write 6 Sent Done Done false
33612024-02-13T21:40:39.864Z 9 Acked 1008 Read 7 Sent Done Done false
33622024-02-13T21:40:39.864Z 10 Acked 1009 Write 5 Sent Done Done false
33632024-02-13T21:40:39.864Z 11 Acked 1010 Write 8 Sent Done Done false
33642024-02-13T21:40:39.864Z 12 Acked 1011 Read 2 Sent Done Done false
33652024-02-13T21:40:39.864Z 13 Acked 1012 Read 9 Sent Done Done false
33662024-02-13T21:40:39.864Z 14 Acked 1013 Read 4 Sent Done Done false
33672024-02-13T21:40:39.865Z 15 Acked 1014 Read 2 Sent Done Done false
33682024-02-13T21:40:39.865Z 16 Acked 1015 Write 4 Sent Done Done false
33692024-02-13T21:40:39.865Z 17 Acked 1016 Write 5 Sent Done Done false
33702024-02-13T21:40:39.865Z 18 Acked 1017 Write 1 Sent Done Done false
33712024-02-13T21:40:39.865Z 19 Acked 1018 Read 1 Sent Done Done false
33722024-02-13T21:40:39.865Z 20 Acked 1019 Read 2 Sent Done Done false
33732024-02-13T21:40:39.865Z 21 Acked 1020 Read 9 Sent Done Done false
33742024-02-13T21:40:39.865Z 22 Acked 1021 Read 6 Sent Done Done false
33752024-02-13T21:40:39.865Z 23 Acked 1022 Write 7 Sent Done Done false
33762024-02-13T21:40:39.866Z 24 Acked 1023 Read 1 Sent Done Done false
33772024-02-13T21:40:39.866Z 25 Acked 1024 Write 9 Sent Done Done false
33782024-02-13T21:40:39.866Z 26 Acked 1025 Read 2 Sent Done Done false
33792024-02-13T21:40:39.866Z 27 Acked 1026 Write 7 Sent Done Done false
33802024-02-13T21:40:39.866Z 28 Acked 1027 Read 10 Sent Done Done false
33812024-02-13T21:40:39.866Z 29 Acked 1028 Read 4 Sent Done Done false
33822024-02-13T21:40:39.866Z 30 Acked 1029 Write 1 Sent Sent Sent false
33832024-02-13T21:40:39.867Z STATES DS:0 DS:1 DS:2 TOTAL
33842024-02-13T21:40:39.867Z New 0 0 0 0
33852024-02-13T21:40:39.867Z Sent 24 1 1 26
33862024-02-13T21:40:39.867Z Done 6 29 29 64
33872024-02-13T21:40:39.868Z Skipped 0 0 0 0
33882024-02-13T21:40:39.868Z Error 0 0 0 0
33892024-02-13T21:40:39.868ZLast Flush: 0 0 0
33902024-02-13T21:40:39.868ZDownstairs last five completed:
33912024-02-13T21:40:39.868ZUpstairs last five completed: 30 29 28 27 26
33922024-02-13T21:40:39.868Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:38.643662148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1255}
33932024-02-13T21:40:39.868ZWrote out file "/tmp/test_repair_verify.data"
33942024-02-13T21:40:39.869Z
33952024-02-13T21:40:39.869ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33962024-02-13T21:40:39.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33972024-02-13T21:40:39.885Z 0 000-019 28 28 28 48 48 48 T T T
33982024-02-13T21:40:39.885Z 1 020-039 22 22 22 39 39 39 F F F
33992024-02-13T21:40:39.885Z 2 040-059 28 28 28 48 48 48 F T T <---
34002024-02-13T21:40:39.885Z 3 060-079 26 26 26 46 46 46 F F F
34012024-02-13T21:40:39.885Z 4 080-099 29 29 29 49 49 49 F T T <---
34022024-02-13T21:40:39.885Z 5 100-119 29 29 29 49 49 49 T T T
34032024-02-13T21:40:39.886Z 6 120-139 29 29 29 49 49 49 F F F
34042024-02-13T21:40:39.886Z 7 140-159 28 28 28 48 48 48 F F F
34052024-02-13T21:40:39.886Z 8 160-179 29 29 29 49 49 49 F F F
34062024-02-13T21:40:39.886Z 9 180-199 27 27 27 47 47 47 F F F
34072024-02-13T21:40:39.886Z 10 200-219 29 29 29 49 49 49 F T T <---
34082024-02-13T21:40:39.886Z 11 220-239 25 25 25 45 45 45 T T T
34092024-02-13T21:40:39.886Z 12 240-259 24 24 24 44 44 44 F T T <---
34102024-02-13T21:40:39.886Z 13 260-279 24 24 24 44 44 44 F T T <---
34112024-02-13T21:40:39.886Z 14 280-299 28 28 28 48 48 48 F T T <---
34122024-02-13T21:40:39.886Z 15 300-319 28 28 28 48 48 48 F T T <---
34132024-02-13T21:40:39.886Z 16 320-339 28 28 28 48 48 48 F T T <---
34142024-02-13T21:40:39.886Z 17 340-359 29 29 29 49 49 49 F T T <---
34152024-02-13T21:40:39.887Z 18 360-379 22 22 22 38 38 38 F F F
34162024-02-13T21:40:39.887Z 19 380-399 22 22 22 38 38 38 F F F
34172024-02-13T21:40:39.887Z 20 400-419 26 26 26 46 46 46 F F F
34182024-02-13T21:40:39.887Z 21 420-439 29 29 29 49 49 49 F F F
34192024-02-13T21:40:39.887Z 22 440-459 25 25 25 45 45 45 F T T <---
34202024-02-13T21:40:39.887Z 23 460-479 26 26 26 46 46 46 F T T <---
34212024-02-13T21:40:39.887Z 24 480-499 27 27 27 47 47 47 F F F
34222024-02-13T21:40:39.887Z 25 500-519 29 29 29 49 49 49 F F F
34232024-02-13T21:40:39.888Z 26 520-539 29 29 29 49 49 49 T T T
34242024-02-13T21:40:39.888Z 27 540-559 29 29 29 49 49 49 T T T
34252024-02-13T21:40:39.888Z 28 560-579 26 26 26 46 46 46 F F F
34262024-02-13T21:40:39.888Z 29 580-599 24 24 24 43 43 43 T T T
34272024-02-13T21:40:39.888ZMax gen: 29, Max flush: 49
34282024-02-13T21:40:39.888ZError: Difference in extent metadata found!
34292024-02-13T21:40:39.888ZA Difference in extent metadata is expected here
34302024-02-13T21:40:39.888ZOn loop 15
34312024-02-13T21:40:39.888Z
34322024-02-13T21:40:39.888ZVerifying data now
34332024-02-13T21:40:45.754ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
34342024-02-13T21:40:45.771Z119a120
34352024-02-13T21:40:45.772Z> 3,
34362024-02-13T21:40:45.772Z122d122
34372024-02-13T21:40:45.772Z< 2,
34382024-02-13T21:40:45.772Z223c223
34392024-02-13T21:40:45.772Z< 4,
34402024-02-13T21:40:45.773Z---
34412024-02-13T21:40:45.773Z> 5,
34422024-02-13T21:40:45.773Z238,240d237
34432024-02-13T21:40:45.773Z< 5,
34442024-02-13T21:40:45.773Z< 5,
34452024-02-13T21:40:45.773Z< 5,
34462024-02-13T21:40:45.774Z241a239,242
34472024-02-13T21:40:45.774Z> 6,
34482024-02-13T21:40:45.774Z> 6,
34492024-02-13T21:40:45.774Z> 7,
34502024-02-13T21:40:45.774Z> 6,
34512024-02-13T21:40:45.774Z244d244
34522024-02-13T21:40:45.775Z< 5,
34532024-02-13T21:40:45.775Zdiff found after verify
34542024-02-13T21:40:45.775ZLoop: 15 Downstairs dump after verify (and repair):
34552024-02-13T21:40:45.785ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34562024-02-13T21:40:45.785Z 0 000-019 30 30 30 50 50 50 F F F
34572024-02-13T21:40:45.785Z 1 020-039 22 22 22 39 39 39 F F F
34582024-02-13T21:40:45.786Z 2 040-059 30 30 30 50 50 50 F F F
34592024-02-13T21:40:45.786Z 3 060-079 26 26 26 46 46 46 F F F
34602024-02-13T21:40:45.786Z 4 080-099 30 30 30 50 50 50 F F F
34612024-02-13T21:40:45.786Z 5 100-119 30 30 30 50 50 50 F F F
34622024-02-13T21:40:45.786Z 6 120-139 29 29 29 49 49 49 F F F
34632024-02-13T21:40:45.787Z 7 140-159 28 28 28 48 48 48 F F F
34642024-02-13T21:40:45.787Z 8 160-179 29 29 29 49 49 49 F F F
34652024-02-13T21:40:45.787Z 9 180-199 27 27 27 47 47 47 F F F
34662024-02-13T21:40:45.787Z 10 200-219 30 30 30 50 50 50 F F F
34672024-02-13T21:40:45.787Z 11 220-239 30 30 30 50 50 50 F F F
34682024-02-13T21:40:45.788Z 12 240-259 30 30 30 50 50 50 F F F
34692024-02-13T21:40:45.788Z 13 260-279 30 30 30 50 50 50 F F F
34702024-02-13T21:40:45.788Z 14 280-299 30 30 30 50 50 50 F F F
34712024-02-13T21:40:45.788Z 15 300-319 30 30 30 50 50 50 F F F
34722024-02-13T21:40:45.788Z 16 320-339 30 30 30 50 50 50 F F F
34732024-02-13T21:40:45.788Z 17 340-359 30 30 30 50 50 50 F F F
34742024-02-13T21:40:45.788Z 18 360-379 22 22 22 38 38 38 F F F
34752024-02-13T21:40:45.789Z 19 380-399 22 22 22 38 38 38 F F F
34762024-02-13T21:40:45.789Z 20 400-419 26 26 26 46 46 46 F F F
34772024-02-13T21:40:45.789Z 21 420-439 29 29 29 49 49 49 F F F
34782024-02-13T21:40:45.789Z 22 440-459 30 30 30 50 50 50 F F F
34792024-02-13T21:40:45.789Z 23 460-479 30 30 30 50 50 50 F F F
34802024-02-13T21:40:45.789Z 24 480-499 27 27 27 47 47 47 F F F
34812024-02-13T21:40:45.790Z 25 500-519 29 29 29 49 49 49 F F F
34822024-02-13T21:40:45.790Z 26 520-539 30 30 30 50 50 50 F F F
34832024-02-13T21:40:45.790Z 27 540-559 30 30 30 50 50 50 F F F
34842024-02-13T21:40:45.790Z 28 560-579 26 26 26 46 46 46 F F F
34852024-02-13T21:40:45.790Z 29 580-599 30 30 30 50 50 50 F F F
34862024-02-13T21:40:45.790ZMax gen: 30, Max flush: 50
34872024-02-13T21:40:45.790Z
34882024-02-13T21:40:45.790ZBegin loop 16
34892024-02-13T21:40:45.790ZDownstairs to restart: 0
34902024-02-13T21:40:45.817Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.608630169Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
34912024-02-13T21:40:45.818Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.608910387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
34922024-02-13T21:40:45.818Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
34932024-02-13T21:40:45.818Z,"time":"2024-02-13T21:40:44.608939527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
34942024-02-13T21:40:45.818Z{"msg":"Crucible 2a244fe7-aad5-44b5-b614-4c754442d3e0 has session id: f3ac0009-687f-4da5-af0a-5639917aed6e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.608971875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
34952024-02-13T21:40:45.819Z{"msg":"Upstairs opts: Upstairs UUID: 2a244fe7-aad5-44b5-b614-4c754442d3e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609021477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
34962024-02-13T21:40:45.819Z{"msg":"Crucible stats registered with UUID: 2a244fe7-aad5-44b5-b614-4c754442d3e0","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609060633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
34972024-02-13T21:40:45.819Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609077956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
34982024-02-13T21:40:45.819Z{"msg":"2a244fe7-aad5-44b5-b614-4c754442d3e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609174631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
34992024-02-13T21:40:45.819Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609207819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35002024-02-13T21:40:45.819Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609223973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35012024-02-13T21:40:45.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609250444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35022024-02-13T21:40:45.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609269377Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35032024-02-13T21:40:45.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609291609Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35042024-02-13T21:40:45.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.60931584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35052024-02-13T21:40:45.821Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609346109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35062024-02-13T21:40:45.821Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609414004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35072024-02-13T21:40:45.822Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609496485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35082024-02-13T21:40:45.822Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609567639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35092024-02-13T21:40:45.822Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609621859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35102024-02-13T21:40:45.822Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.609649809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"io task","client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35112024-02-13T21:40:45.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.610259158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35122024-02-13T21:40:45.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.610289707Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35132024-02-13T21:40:45.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.610304312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35142024-02-13T21:40:45.823Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.610331482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35152024-02-13T21:40:45.823Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.610871396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35162024-02-13T21:40:45.823Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611015634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35172024-02-13T21:40:45.824Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611117917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35182024-02-13T21:40:45.824Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611148255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35192024-02-13T21:40:45.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611322472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35202024-02-13T21:40:45.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611370025Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35212024-02-13T21:40:45.825Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611399704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35222024-02-13T21:40:45.825Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611427054Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35232024-02-13T21:40:45.825Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611459382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35242024-02-13T21:40:45.825Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611474577Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35252024-02-13T21:40:45.826Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.611682042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35262024-02-13T21:40:45.826Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612070881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35272024-02-13T21:40:45.826Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612098171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35282024-02-13T21:40:45.826Z{"msg":"[0]R flush_numbers[0..12]: [50, 39, 50, 46, 50, 50, 49, 48, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612124181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35292024-02-13T21:40:45.826Z{"msg":"[0]R generation[0..12]: [30, 22, 30, 26, 30, 30, 29, 28, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612152361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35302024-02-13T21:40:45.827Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612167785Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35312024-02-13T21:40:45.827Z{"msg":"[1]R flush_numbers[0..12]: [50, 39, 50, 46, 50, 50, 49, 48, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.61218294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35322024-02-13T21:40:45.827Z{"msg":"[1]R generation[0..12]: [30, 22, 30, 26, 30, 30, 29, 28, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612197565Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35332024-02-13T21:40:45.827Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612219667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35342024-02-13T21:40:45.827Z{"msg":"[2]R flush_numbers[0..12]: [50, 39, 50, 46, 50, 50, 49, 48, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612234201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35352024-02-13T21:40:45.828Z{"msg":"[2]R generation[0..12]: [30, 22, 30, 26, 30, 30, 29, 28, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612247996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35362024-02-13T21:40:45.828Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612262161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35372024-02-13T21:40:45.828Z{"msg":"Max found gen is 31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612282464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35382024-02-13T21:40:45.828ZWait for a query_work_queue command to finish before sending IO
35392024-02-13T21:40:45.828Z{"msg":"Generation requested: 32 >= found:31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612300448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35402024-02-13T21:40:45.828Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612314052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35412024-02-13T21:40:45.829Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612334475Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"":"downstairs","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35422024-02-13T21:40:45.829Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.61234839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35432024-02-13T21:40:45.829Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612360386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"0","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35442024-02-13T21:40:45.829Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612379969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"1","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35452024-02-13T21:40:45.829Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612393454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"client":"2","":"downstairs","session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35462024-02-13T21:40:45.829Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612406819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35472024-02-13T21:40:45.830Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612425582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35482024-02-13T21:40:45.830Z{"msg":"2a244fe7-aad5-44b5-b614-4c754442d3e0 is now active with session: f3ac0009-687f-4da5-af0a-5639917aed6e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612439037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35492024-02-13T21:40:45.830Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612450853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266,"session_id":"f3ac0009-687f-4da5-af0a-5639917aed6e"}
35502024-02-13T21:40:45.830Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:44.612463218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
35512024-02-13T21:40:45.830ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35522024-02-13T21:40:45.830ZLoading write count information from file "/tmp/test_repair_verify.data"
35532024-02-13T21:40:45.831ZRun Repair workload
35542024-02-13T21:40:45.831Z01/30 Write block 351 len 40960 data: 2 2 3 3 3 4 4 3 3 3
35552024-02-13T21:40:45.831Z02/30 Read block 293 len 8192
35562024-02-13T21:40:45.832Z03/30 Read block 75 len 36864
35572024-02-13T21:40:45.847Z04/30 Write block 26 len 16384 data: 4 2 2 2
35582024-02-13T21:40:45.847Z05/30 Write block 256 len 12288 data: 4 4 4
35592024-02-13T21:40:45.847Z06/30 Read block 415 len 32768
35602024-02-13T21:40:45.848Z07/30 Read block 462 len 32768
35612024-02-13T21:40:45.848Z08/30 Write block 140 len 32768 data: 2 3 4 4 4 4 4 4
35622024-02-13T21:40:45.848Z09/30 Read block 215 len 40960
35632024-02-13T21:40:45.848Z10/30 Read block 82 len 40960
35642024-02-13T21:40:45.849Z11/30 Read block 488 len 8192
35652024-02-13T21:40:45.849Z12/30 Write block 131 len 20480 data: 2 2 2 2 3
35662024-02-13T21:40:45.849Z13/30 Read block 425 len 12288
35672024-02-13T21:40:45.849Z14/30 Write block 322 len 24576 data: 2 2 2 3 3 3
35682024-02-13T21:40:45.849Z15/30 Write block 411 len 40960 data: 2 2 3 4 4 5 6 6 6 5
35692024-02-13T21:40:45.850Z16/30 Write block 0 len 8192 data: 2 2
35702024-02-13T21:40:45.850Z17/30 Read block 104 len 8192
35712024-02-13T21:40:45.850Z18/30 Flush
35722024-02-13T21:40:45.850Z19/30 Write block 382 len 8192 data: 7 6
35732024-02-13T21:40:45.850Z20/30 Flush
35742024-02-13T21:40:45.850Z21/30 Write block 54 len 36864 data: 3 3 4 4 5 5 5 5 5
35752024-02-13T21:40:45.850Z22/30 Write block 277 len 16384 data: 3 3 3 3
35762024-02-13T21:40:45.850Z23/30 Read block 337 len 24576
35772024-02-13T21:40:45.851Z24/30 Write block 493 len 28672 data: 4 6 6 5 5 6 5
35782024-02-13T21:40:45.851Z25/30 Read block 245 len 4096
35792024-02-13T21:40:45.851Z26/30 Read block 291 len 36864
35802024-02-13T21:40:45.851Z27/30 Read block 182 len 4096
35812024-02-13T21:40:45.851Z28/30 Write block 131 len 32768 data: 3 3 3 3 4 3 3 3
35822024-02-13T21:40:45.851Z29/30 Read block 28 len 24576
35832024-02-13T21:40:45.851Z30/30 Read block 122 len 20480
35842024-02-13T21:40:45.851Z----------------------------------------------------------------
35852024-02-13T21:40:45.851Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35862024-02-13T21:40:45.851ZStates: Active Active Active
35872024-02-13T21:40:45.851ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35882024-02-13T21:40:45.852Z 1 Acked 1000 Write 10 Sent Done Done false
35892024-02-13T21:40:45.852Z 2 Acked 1001 Read 2 Sent Done Done false
35902024-02-13T21:40:45.852Z 3 Acked 1002 Read 9 Sent Done Done false
35912024-02-13T21:40:45.852Z 4 Acked 1003 Write 4 Sent Done Done false
35922024-02-13T21:40:45.852Z 5 Acked 1004 Write 3 Sent Done Done false
35932024-02-13T21:40:45.852Z 6 Acked 1005 Read 8 Sent Done Done false
35942024-02-13T21:40:45.852Z 7 Acked 1006 Read 8 Sent Done Done false
35952024-02-13T21:40:45.852Z 8 Acked 1007 Write 8 Sent Done Done false
35962024-02-13T21:40:45.852Z 9 Acked 1008 Read 10 Sent Done Done false
35972024-02-13T21:40:45.852Z 10 Acked 1009 Read 10 Sent Done Done false
35982024-02-13T21:40:45.852Z 11 Acked 1010 Read 2 Sent Done Done false
35992024-02-13T21:40:45.852Z 12 Acked 1011 Write 5 Sent Done Done false
36002024-02-13T21:40:45.852Z 13 Acked 1012 Read 3 Sent Done Done false
36012024-02-13T21:40:45.852Z 14 Acked 1013 Write 6 Sent Done Done false
36022024-02-13T21:40:45.853Z 15 Acked 1014 Write 10 Sent Done Done false
36032024-02-13T21:40:45.853Z 16 Acked 1015 Write 2 Sent Done Done false
36042024-02-13T21:40:45.853Z 17 Acked 1016 Read 2 Sent Done Done false
36052024-02-13T21:40:45.853Z 18 Acked 1017 Flush 0 Sent Done Done false
36062024-02-13T21:40:45.853Z 19 Acked 1018 Write 2 Sent Done Done false
36072024-02-13T21:40:45.853Z 20 Acked 1019 Flush 0 Sent Done Done false
36082024-02-13T21:40:45.853Z 21 Acked 1020 Write 9 Sent Done Done false
36092024-02-13T21:40:45.853Z 22 Acked 1021 Write 4 Sent Done Done false
36102024-02-13T21:40:45.853Z 23 Acked 1022 Read 6 Sent Done Done false
36112024-02-13T21:40:45.853Z 24 Acked 1023 Write 7 Sent Done Done false
36122024-02-13T21:40:45.853Z 25 Acked 1024 Read 1 Sent Done Done false
36132024-02-13T21:40:45.853Z 26 Acked 1025 Read 9 Sent Done Done false
36142024-02-13T21:40:45.853Z 27 Acked 1026 Read 1 Sent Done Done false
36152024-02-13T21:40:45.854Z 28 Acked 1027 Write 8 Sent Done Done false
36162024-02-13T21:40:45.854Z 29 Acked 1028 Read 6 Sent Done Done false
36172024-02-13T21:40:45.854Z 30 Acked 1029 Read 5 Sent Sent Done false
36182024-02-13T21:40:45.854Z STATES DS:0 DS:1 DS:2 TOTAL
36192024-02-13T21:40:45.854Z New 0 0 0 0
36202024-02-13T21:40:45.854Z Sent 30 1 0 31
36212024-02-13T21:40:45.854Z Done 0 29 30 59
36222024-02-13T21:40:45.854Z Skipped 0 0 0 0
36232024-02-13T21:40:45.854Z Error 0 0 0 0
36242024-02-13T21:40:45.854ZLast Flush: 0 1019 1019
36252024-02-13T21:40:45.854ZDownstairs last five completed:
36262024-02-13T21:40:45.854ZUpstairs last five completed: 30 29 28 27 26
36272024-02-13T21:40:45.854Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:44.644221569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1266}
36282024-02-13T21:40:45.855ZWrote out file "/tmp/test_repair_verify.data"
36292024-02-13T21:40:45.855Z
36302024-02-13T21:40:45.855ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36312024-02-13T21:40:45.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36322024-02-13T21:40:45.868Z 0 000-019 30 32 32 50 51 51 F F F <---
36332024-02-13T21:40:45.868Z 1 020-039 22 32 32 39 51 51 F F F <---
36342024-02-13T21:40:45.868Z 2 040-059 30 30 30 50 50 50 F T T <---
36352024-02-13T21:40:45.868Z 3 060-079 26 26 26 46 46 46 F T T <---
36362024-02-13T21:40:45.868Z 4 080-099 30 30 30 50 50 50 F F F
36372024-02-13T21:40:45.868Z 5 100-119 30 30 30 50 50 50 F F F
36382024-02-13T21:40:45.868Z 6 120-139 29 32 32 49 51 51 F T T <---
36392024-02-13T21:40:45.868Z 7 140-159 28 32 32 48 51 51 F F F <---
36402024-02-13T21:40:45.869Z 8 160-179 29 29 29 49 49 49 F F F
36412024-02-13T21:40:45.869Z 9 180-199 27 27 27 47 47 47 F F F
36422024-02-13T21:40:45.869Z 10 200-219 30 30 30 50 50 50 F F F
36432024-02-13T21:40:45.869Z 11 220-239 30 30 30 50 50 50 F F F
36442024-02-13T21:40:45.869Z 12 240-259 30 32 32 50 51 51 F F F <---
36452024-02-13T21:40:45.869Z 13 260-279 30 30 30 50 50 50 F T T <---
36462024-02-13T21:40:45.869Z 14 280-299 30 30 30 50 50 50 F T T <---
36472024-02-13T21:40:45.869Z 15 300-319 30 30 30 50 50 50 F F F
36482024-02-13T21:40:45.869Z 16 320-339 30 32 32 50 51 51 F F F <---
36492024-02-13T21:40:45.869Z 17 340-359 30 32 32 50 51 51 F F F <---
36502024-02-13T21:40:45.869Z 18 360-379 22 32 32 38 51 51 F F F <---
36512024-02-13T21:40:45.869Z 19 380-399 22 32 32 38 52 52 F F F <---
36522024-02-13T21:40:45.869Z 20 400-419 26 32 32 46 51 51 F F F <---
36532024-02-13T21:40:45.870Z 21 420-439 29 32 32 49 51 51 F F F <---
36542024-02-13T21:40:45.870Z 22 440-459 30 30 30 50 50 50 F F F
36552024-02-13T21:40:45.870Z 23 460-479 30 30 30 50 50 50 F F F
36562024-02-13T21:40:45.870Z 24 480-499 27 27 27 47 47 47 F T T <---
36572024-02-13T21:40:45.870Z 25 500-519 29 29 29 49 49 49 F F F
36582024-02-13T21:40:45.870Z 26 520-539 30 30 30 50 50 50 F F F
36592024-02-13T21:40:45.870Z 27 540-559 30 30 30 50 50 50 F F F
36602024-02-13T21:40:45.870Z 28 560-579 26 26 26 46 46 46 F F F
36612024-02-13T21:40:45.870Z 29 580-599 30 30 30 50 50 50 F F F
36622024-02-13T21:40:45.870ZMax gen: 32, Max flush: 52
36632024-02-13T21:40:45.870ZError: Difference in extent metadata found!
36642024-02-13T21:40:45.871ZA Difference in extent metadata is expected here
36652024-02-13T21:40:45.871ZOn loop 16
36662024-02-13T21:40:45.871Z
36672024-02-13T21:40:45.871ZVerifying data now
36682024-02-13T21:40:50.605ZNo change after verify
36692024-02-13T21:40:50.605ZLoop: 16 Downstairs dump after verify (and repair):
36702024-02-13T21:40:50.620ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36712024-02-13T21:40:50.620Z 0 000-019 32 32 32 51 51 51 F F F
36722024-02-13T21:40:50.621Z 1 020-039 32 32 32 51 51 51 F F F
36732024-02-13T21:40:50.621Z 2 040-059 33 33 33 53 53 53 F F F
36742024-02-13T21:40:50.621Z 3 060-079 33 33 33 53 53 53 F F F
36752024-02-13T21:40:50.621Z 4 080-099 30 30 30 50 50 50 F F F
36762024-02-13T21:40:50.621Z 5 100-119 30 30 30 50 50 50 F F F
36772024-02-13T21:40:50.621Z 6 120-139 33 33 33 53 53 53 F F F
36782024-02-13T21:40:50.621Z 7 140-159 32 32 32 51 51 51 F F F
36792024-02-13T21:40:50.621Z 8 160-179 29 29 29 49 49 49 F F F
36802024-02-13T21:40:50.625Z 9 180-199 27 27 27 47 47 47 F F F
36812024-02-13T21:40:50.625Z 10 200-219 30 30 30 50 50 50 F F F
36822024-02-13T21:40:50.625Z 11 220-239 30 30 30 50 50 50 F F F
36832024-02-13T21:40:50.625Z 12 240-259 32 32 32 51 51 51 F F F
36842024-02-13T21:40:50.625Z 13 260-279 33 33 33 53 53 53 F F F
36852024-02-13T21:40:50.625Z 14 280-299 33 33 33 53 53 53 F F F
36862024-02-13T21:40:50.625Z 15 300-319 30 30 30 50 50 50 F F F
36872024-02-13T21:40:50.626Z 16 320-339 32 32 32 51 51 51 F F F
36882024-02-13T21:40:50.626Z 17 340-359 32 32 32 51 51 51 F F F
36892024-02-13T21:40:50.626Z 18 360-379 32 32 32 51 51 51 F F F
36902024-02-13T21:40:50.626Z 19 380-399 32 32 32 52 52 52 F F F
36912024-02-13T21:40:50.626Z 20 400-419 32 32 32 51 51 51 F F F
36922024-02-13T21:40:50.626Z 21 420-439 32 32 32 51 51 51 F F F
36932024-02-13T21:40:50.626Z 22 440-459 30 30 30 50 50 50 F F F
36942024-02-13T21:40:50.626Z 23 460-479 30 30 30 50 50 50 F F F
36952024-02-13T21:40:50.626Z 24 480-499 33 33 33 53 53 53 F F F
36962024-02-13T21:40:50.626Z 25 500-519 29 29 29 49 49 49 F F F
36972024-02-13T21:40:50.626Z 26 520-539 30 30 30 50 50 50 F F F
36982024-02-13T21:40:50.626Z 27 540-559 30 30 30 50 50 50 F F F
36992024-02-13T21:40:50.626Z 28 560-579 26 26 26 46 46 46 F F F
37002024-02-13T21:40:50.626Z 29 580-599 30 30 30 50 50 50 F F F
37012024-02-13T21:40:50.626ZMax gen: 33, Max flush: 53
37022024-02-13T21:40:50.626Z
37032024-02-13T21:40:50.626ZBegin loop 17
37042024-02-13T21:40:50.626ZDownstairs to restart: 0
37052024-02-13T21:40:50.652Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458237213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
37062024-02-13T21:40:50.652Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name"Crucible runtime is spawned
37072024-02-13T21:40:50.653Z:"crucible","level":30,"time":"2024-02-13T21:40:49.45857486Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
37082024-02-13T21:40:50.653Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458626141Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
37092024-02-13T21:40:50.653Z{"msg":"Crucible a8f244a0-b4ba-4c00-b455-9dbf1b372c29 has session id: 1bffd65e-a216-4baa-93c6-56383f909a83","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458645235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37102024-02-13T21:40:50.653Z{"msg":"Upstairs opts: Upstairs UUID: a8f244a0-b4ba-4c00-b455-9dbf1b372c29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458672725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37112024-02-13T21:40:50.653Z{"msg":"Crucible stats registered with UUID: a8f244a0-b4ba-4c00-b455-9dbf1b372c29","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458700135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37122024-02-13T21:40:50.653Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458716809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
37132024-02-13T21:40:50.653Z{"msg":"a8f244a0-b4ba-4c00-b455-9dbf1b372c29 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458746108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37142024-02-13T21:40:50.653Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458762102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37152024-02-13T21:40:50.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458804217Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37162024-02-13T21:40:50.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458827049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37172024-02-13T21:40:50.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458864115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37182024-02-13T21:40:50.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458896394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37192024-02-13T21:40:50.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458944916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37202024-02-13T21:40:50.654Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.458976134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37212024-02-13T21:40:50.654Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459023408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37222024-02-13T21:40:50.654Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459064833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37232024-02-13T21:40:50.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459090143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37242024-02-13T21:40:50.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459115424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37252024-02-13T21:40:50.654Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459151911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"io task","client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37262024-02-13T21:40:50.654Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459802885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37272024-02-13T21:40:50.655Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459860884Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37282024-02-13T21:40:50.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459899939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37292024-02-13T21:40:50.655Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.459945333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37302024-02-13T21:40:50.655Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.460409515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37312024-02-13T21:40:50.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.460540267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37322024-02-13T21:40:50.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.460841168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37332024-02-13T21:40:50.655Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.460877205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37342024-02-13T21:40:50.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461004799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37352024-02-13T21:40:50.655Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461071105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37362024-02-13T21:40:50.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461098875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37372024-02-13T21:40:50.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.46114012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37382024-02-13T21:40:50.656Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461184983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37392024-02-13T21:40:50.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461224329Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37402024-02-13T21:40:50.656Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461363548Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37412024-02-13T21:40:50.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461760015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37422024-02-13T21:40:50.656Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461798371Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37432024-02-13T21:40:50.656Z{"msg":"[0]R flush_numbers[0..12]: [51, 51, 53, 53, 50, 50, 53, 51, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461814635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37442024-02-13T21:40:50.656Z{"msg":"[0]R generation[0..12]: [32, 32, 33, 33, 30, 30, 33, 32, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461836307Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37452024-02-13T21:40:50.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461851102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37462024-02-13T21:40:50.656Z{"msg":"[1]R flush_numbers[0..12]: [51, 51, 53, 53, 50, 50, 53, 51, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461872314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37472024-02-13T21:40:50.656Z{"msg":"[1]R generation[0..12]: [32, 32, 33, 33, 30, 30, 33, 32, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461886219Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37482024-02-13T21:40:50.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461900174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37492024-02-13T21:40:50.657Z{"msg":"[2]R flush_numbers[0..12]: [51, 51, 53, 53, 50, 50, 53, 51, 49, 47, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461914099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37502024-02-13T21:40:50.657ZWait for a query_work_queue command to finish before sending IO
37512024-02-13T21:40:50.657Z{"msg":"[2]R generation[0..12]: [32, 32, 33, 33, 30, 30, 33, 32, 29, 27, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.46193824Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37522024-02-13T21:40:50.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461953315Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37532024-02-13T21:40:50.657Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461973367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37542024-02-13T21:40:50.657Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.461987532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37552024-02-13T21:40:50.657Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462012293Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37562024-02-13T21:40:50.657Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462025998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"":"downstairs","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37572024-02-13T21:40:50.657Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462039963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37582024-02-13T21:40:50.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462059026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"0","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37592024-02-13T21:40:50.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462072631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"1","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37602024-02-13T21:40:50.658Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462092004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"client":"2","":"downstairs","session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37612024-02-13T21:40:50.658Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462105359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37622024-02-13T21:40:50.658Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462117755Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37632024-02-13T21:40:50.658Z{"msg":"a8f244a0-b4ba-4c00-b455-9dbf1b372c29 is now active with session: 1bffd65e-a216-4baa-93c6-56383f909a83","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462139037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37642024-02-13T21:40:50.658Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462151123Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274,"session_id":"1bffd65e-a216-4baa-93c6-56383f909a83"}
37652024-02-13T21:40:50.658Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:49.462170086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
37662024-02-13T21:40:50.658ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37672024-02-13T21:40:50.658ZLoading write count information from file "/tmp/test_repair_verify.data"
37682024-02-13T21:40:50.658ZRun Repair workload
37692024-02-13T21:40:50.658Z01/30 Flush
37702024-02-13T21:40:50.659Z02/30 Write block 387 len 32768 data: 4 3 2 2 2 2 2 2
37712024-02-13T21:40:50.659Z03/30 Write block 506 len 24576 data: 4 4 4 5 6 6
37722024-02-13T21:40:50.659Z04/30 Write block 185 len 36864 data: 7 6 4 6 6 6 6 6 6
37732024-02-13T21:40:50.666Z05/30 Write block 155 len 16384 data: 3 3 4 4
37742024-02-13T21:40:50.666Z06/30 Read block 183 len 28672
37752024-02-13T21:40:50.666Z07/30 Flush
37762024-02-13T21:40:50.666Z08/30 Write block 517 len 28672 data: 6 6 6 6 6 5 4
37772024-02-13T21:40:50.667Z09/30 Read block 474 len 20480
37782024-02-13T21:40:50.667Z10/30 Write block 27 len 40960 data: 3 3 3 2 4 4 3 4 5 5
37792024-02-13T21:40:50.667Z11/30 Flush
37802024-02-13T21:40:50.667Z12/30 Write block 590 len 12288 data: 4 4 4
37812024-02-13T21:40:50.667Z13/30 Read block 158 len 32768
37822024-02-13T21:40:50.667Z14/30 Read block 381 len 8192
37832024-02-13T21:40:50.667Z15/30 Read block 11 len 24576
37842024-02-13T21:40:50.667Z16/30 Write block 6 len 36864 data: 3 3 3 3 2 4 4 4 4
37852024-02-13T21:40:50.668Z17/30 Write block 116 len 28672 data: 3 3 3 3 4 5 5
37862024-02-13T21:40:50.668Z18/30 Flush
37872024-02-13T21:40:50.672Z19/30 Write block 367 len 36864 data: 4 3 3 3 3 2 3 3 5
37882024-02-13T21:40:50.672Z20/30 Write block 311 len 8192 data: 4 4
37892024-02-13T21:40:50.672Z21/30 Write block 271 len 36864 data: 3 3 3 3 3 3 4 4 4
37902024-02-13T21:40:50.672Z22/30 Read block 30 len 36864
37912024-02-13T21:40:50.672Z23/30 Read block 376 len 16384
37922024-02-13T21:40:50.672Z24/30 Read block 392 len 28672
37932024-02-13T21:40:50.672Z25/30 Flush
37942024-02-13T21:40:50.676Z26/30 Write block 29 len 36864 data: 4 3 5 5 4 5 6 6 4
37952024-02-13T21:40:50.677Z27/30 Read block 250 len 32768
37962024-02-13T21:40:50.677Z28/30 Read block 341 len 28672
37972024-02-13T21:40:50.677Z29/30 Write block 425 len 32768 data: 4 4 3 4 4 4 3 4
37982024-02-13T21:40:50.687Z30/30 Read block 249 len 20480
37992024-02-13T21:40:50.687Z----------------------------------------------------------------
38002024-02-13T21:40:50.687Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
38012024-02-13T21:40:50.687ZStates: Active Active Active
38022024-02-13T21:40:50.687ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38032024-02-13T21:40:50.687Z 1 Acked 1000 Flush 0 Sent Done Done false
38042024-02-13T21:40:50.688Z 2 Acked 1001 Write 8 Sent Done Done false
38052024-02-13T21:40:50.688Z 3 Acked 1002 Write 6 Sent Done Done false
38062024-02-13T21:40:50.688Z 4 Acked 1003 Write 9 Sent Done Done false
38072024-02-13T21:40:50.688Z 5 Acked 1004 Write 4 Sent Done Done false
38082024-02-13T21:40:50.688Z 6 Acked 1005 Read 7 Sent Done Done false
38092024-02-13T21:40:50.688Z 7 Acked 1006 Flush 0 Sent Done Done false
38102024-02-13T21:40:50.688Z 8 Acked 1007 Write 7 Sent Done Done false
38112024-02-13T21:40:50.688Z 9 Acked 1008 Read 5 Sent Done Done false
38122024-02-13T21:40:50.688Z 10 Acked 1009 Write 10 Sent Done Done false
38132024-02-13T21:40:50.688Z 11 Acked 1010 Flush 0 Sent Done Done false
38142024-02-13T21:40:50.689Z 12 Acked 1011 Write 3 Sent Done Done false
38152024-02-13T21:40:50.689Z 13 Acked 1012 Read 8 Sent Done Done false
38162024-02-13T21:40:50.689Z 14 Acked 1013 Read 2 Sent Done Done false
38172024-02-13T21:40:50.689Z 15 Acked 1014 Read 6 Sent Done Done false
38182024-02-13T21:40:50.689Z 16 Acked 1015 Write 9 Sent Done Done false
38192024-02-13T21:40:50.689Z 17 Acked 1016 Write 7 Sent Done Done false
38202024-02-13T21:40:50.689Z 18 Acked 1017 Flush 0 Sent Done Done false
38212024-02-13T21:40:50.689Z 19 Acked 1018 Write 9 Sent Done Done false
38222024-02-13T21:40:50.690Z 20 Acked 1019 Write 2 Sent Done Done false
38232024-02-13T21:40:50.690Z 21 Acked 1020 Write 9 Sent Done Done false
38242024-02-13T21:40:50.690Z 22 Acked 1021 Read 9 Sent Done Done false
38252024-02-13T21:40:50.690Z 23 Acked 1022 Read 4 Sent Done Done false
38262024-02-13T21:40:50.690Z 24 Acked 1023 Read 7 Sent Done Done false
38272024-02-13T21:40:50.690Z 25 Acked 1024 Flush 0 Sent Done Done false
38282024-02-13T21:40:50.690Z 26 Acked 1025 Write 9 Sent Done Done false
38292024-02-13T21:40:50.690Z 27 Acked 1026 Read 8 Sent Done Done false
38302024-02-13T21:40:50.691Z 28 Acked 1027 Read 7 Sent Done Done false
38312024-02-13T21:40:50.691Z 29 Acked 1028 Write 8 Sent Done Done false
38322024-02-13T21:40:50.691Z 30 Acked 1029 Read 5 Sent Done Sent false
38332024-02-13T21:40:50.691Z STATES DS:0 DS:1 DS:2 TOTAL
38342024-02-13T21:40:50.691Z New 0 0 0 0
38352024-02-13T21:40:50.691Z Sent 30 0 1 31
38362024-02-13T21:40:50.691Z Done 0 30 29 59
38372024-02-13T21:40:50.691Z Skipped 0 0 0 0
38382024-02-13T21:40:50.692Z Error 0 0 0 0
38392024-02-13T21:40:50.692ZLast Flush: 0 1024 1024
38402024-02-13T21:40:50.692ZDownstairs last five completed:
38412024-02-13T21:40:50.692ZUpstairs last five completed: 30 29 28 27 26
38422024-02-13T21:40:50.692Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:49.497273313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1274}
38432024-02-13T21:40:50.692ZWrote out file "/tmp/test_repair_verify.data"
38442024-02-13T21:40:50.696Z
38452024-02-13T21:40:50.696ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38462024-02-13T21:40:50.701ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38472024-02-13T21:40:50.701Z 0 000-019 32 34 34 51 57 57 F F F <---
38482024-02-13T21:40:50.701Z 1 020-039 32 34 34 51 56 56 F T T <---
38492024-02-13T21:40:50.701Z 2 040-059 33 33 33 53 53 53 F F F
38502024-02-13T21:40:50.702Z 3 060-079 33 33 33 53 53 53 F F F
38512024-02-13T21:40:50.702Z 4 080-099 30 30 30 50 50 50 F F F
38522024-02-13T21:40:50.702Z 5 100-119 30 34 34 50 57 57 F F F <---
38532024-02-13T21:40:50.702Z 6 120-139 33 34 34 53 57 57 F F F <---
38542024-02-13T21:40:50.702Z 7 140-159 32 34 34 51 55 55 F F F <---
38552024-02-13T21:40:50.702Z 8 160-179 29 29 29 49 49 49 F F F
38562024-02-13T21:40:50.702Z 9 180-199 27 34 34 47 55 55 F F F <---
38572024-02-13T21:40:50.702Z 10 200-219 30 30 30 50 50 50 F F F
38582024-02-13T21:40:50.702Z 11 220-239 30 30 30 50 50 50 F F F
38592024-02-13T21:40:50.702Z 12 240-259 32 32 32 51 51 51 F F F
38602024-02-13T21:40:50.703Z 13 260-279 33 34 34 53 58 58 F F F <---
38612024-02-13T21:40:50.703Z 14 280-299 33 33 33 53 53 53 F F F
38622024-02-13T21:40:50.703Z 15 300-319 30 34 34 50 58 58 F F F <---
38632024-02-13T21:40:50.703Z 16 320-339 32 32 32 51 51 51 F F F
38642024-02-13T21:40:50.703Z 17 340-359 32 32 32 51 51 51 F F F
38652024-02-13T21:40:50.703Z 18 360-379 32 34 34 51 58 58 F F F <---
38662024-02-13T21:40:50.703Z 19 380-399 32 34 34 52 55 55 F F F <---
38672024-02-13T21:40:50.703Z 20 400-419 32 32 32 51 51 51 F F F
38682024-02-13T21:40:50.703Z 21 420-439 32 32 32 51 51 51 F T T <---
38692024-02-13T21:40:50.703Z 22 440-459 30 30 30 50 50 50 F F F
38702024-02-13T21:40:50.703Z 23 460-479 30 30 30 50 50 50 F F F
38712024-02-13T21:40:50.703Z 24 480-499 33 33 33 53 53 53 F F F
38722024-02-13T21:40:50.704Z 25 500-519 29 34 34 49 56 56 F F F <---
38732024-02-13T21:40:50.704Z 26 520-539 30 34 34 50 56 56 F F F <---
38742024-02-13T21:40:50.704Z 27 540-559 30 30 30 50 50 50 F F F
38752024-02-13T21:40:50.704Z 28 560-579 26 26 26 46 46 46 F F F
38762024-02-13T21:40:50.704Z 29 580-599 30 34 34 50 57 57 F F F <---
38772024-02-13T21:40:50.704ZMax gen: 34, Max flush: 58
38782024-02-13T21:40:50.708ZError: Difference in extent metadata found!
38792024-02-13T21:40:50.708ZA Difference in extent metadata is expected here
38802024-02-13T21:40:50.708ZOn loop 17
38812024-02-13T21:40:50.708Z
38822024-02-13T21:40:50.708ZVerifying data now
38832024-02-13T21:40:55.987ZNo change after verify
38842024-02-13T21:40:55.987ZLoop: 17 Downstairs dump after verify (and repair):
38852024-02-13T21:40:55.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38862024-02-13T21:40:55.987Z 0 000-019 34 34 34 57 57 57 F F F
38872024-02-13T21:40:55.988Z 1 020-039 35 35 35 59 59 59 F F F
38882024-02-13T21:40:55.988Z 2 040-059 33 33 33 53 53 53 F F F
38892024-02-13T21:40:55.988Z 3 060-079 33 33 33 53 53 53 F F F
38902024-02-13T21:40:55.988Z 4 080-099 30 30 30 50 50 50 F F F
38912024-02-13T21:40:55.988Z 5 100-119 34 34 34 57 57 57 F F F
38922024-02-13T21:40:55.988Z 6 120-139 34 34 34 57 57 57 F F F
38932024-02-13T21:40:55.988Z 7 140-159 34 34 34 55 55 55 F F F
38942024-02-13T21:40:55.988Z 8 160-179 29 29 29 49 49 49 F F F
38952024-02-13T21:40:55.989Z 9 180-199 34 34 34 55 55 55 F F F
38962024-02-13T21:40:55.989Z 10 200-219 30 30 30 50 50 50 F F F
38972024-02-13T21:40:55.989Z 11 220-239 30 30 30 50 50 50 F F F
38982024-02-13T21:40:55.989Z 12 240-259 32 32 32 51 51 51 F F F
38992024-02-13T21:40:55.989Z 13 260-279 34 34 34 58 58 58 F F F
39002024-02-13T21:40:55.989Z 14 280-299 33 33 33 53 53 53 F F F
39012024-02-13T21:40:55.989Z 15 300-319 34 34 34 58 58 58 F F F
39022024-02-13T21:40:55.989Z 16 320-339 32 32 32 51 51 51 F F F
39032024-02-13T21:40:55.989Z 17 340-359 32 32 32 51 51 51 F F F
39042024-02-13T21:40:55.990Z 18 360-379 34 34 34 58 58 58 F F F
39052024-02-13T21:40:55.990Z 19 380-399 34 34 34 55 55 55 F F F
39062024-02-13T21:40:55.990Z 20 400-419 32 32 32 51 51 51 F F F
39072024-02-13T21:40:55.990Z 21 420-439 35 35 35 59 59 59 F F F
39082024-02-13T21:40:55.990Z 22 440-459 30 30 30 50 50 50 F F F
39092024-02-13T21:40:55.990Z 23 460-479 30 30 30 50 50 50 F F F
39102024-02-13T21:40:55.990Z 24 480-499 33 33 33 53 53 53 F F F
39112024-02-13T21:40:55.990Z 25 500-519 34 34 34 56 56 56 F F F
39122024-02-13T21:40:55.991Z 26 520-539 34 34 34 56 56 56 F F F
39132024-02-13T21:40:55.991Z 27 540-559 30 30 30 50 50 50 F F F
39142024-02-13T21:40:55.991Z 28 560-579 26 26 26 46 46 46 F F F
39152024-02-13T21:40:55.991Z 29 580-599 34 34 34 57 57 57 F F F
39162024-02-13T21:40:55.991ZMax gen: 35, Max flush: 59
39172024-02-13T21:40:55.991Z
39182024-02-13T21:40:55.991ZBegin loop 18
39192024-02-13T21:40:55.991ZDownstairs to restart: 0
39202024-02-13T21:40:56.000Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.218796996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
39212024-02-13T21:40:56.000Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219076894Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
39222024-02-13T21:40:56.001ZCrucible runtime is spawned
39232024-02-13T21:40:56.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.21914459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
39242024-02-13T21:40:56.001Z{"msg":"Crucible 96487f7e-a1ef-4923-bea7-745cfee104ad has session id: 7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219187924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39252024-02-13T21:40:56.001Z{"msg":"Upstairs opts: Upstairs UUID: 96487f7e-a1ef-4923-bea7-745cfee104ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219240245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39262024-02-13T21:40:56.001Z{"msg":"Crucible stats registered with UUID: 96487f7e-a1ef-4923-bea7-745cfee104ad","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219277562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39272024-02-13T21:40:56.001Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219301843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
39282024-02-13T21:40:56.001Z{"msg":"96487f7e-a1ef-4923-bea7-745cfee104ad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219315828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39292024-02-13T21:40:56.001Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219335211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39302024-02-13T21:40:56.001Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219355863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39312024-02-13T21:40:56.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219375986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39322024-02-13T21:40:56.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219407794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39332024-02-13T21:40:56.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219444261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39342024-02-13T21:40:56.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219481588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39352024-02-13T21:40:56.002Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219517125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39362024-02-13T21:40:56.002Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219571825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39372024-02-13T21:40:56.002Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219610231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39382024-02-13T21:40:56.002Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219637751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39392024-02-13T21:40:56.002Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219663572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39402024-02-13T21:40:56.002Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.219690362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"io task","client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39412024-02-13T21:40:56.002Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.22035819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39422024-02-13T21:40:56.002Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.220424176Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39432024-02-13T21:40:56.002Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.220473308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39442024-02-13T21:40:56.003Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.220532137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39452024-02-13T21:40:56.003Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221052458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39462024-02-13T21:40:56.003Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221168046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39472024-02-13T21:40:56.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221421094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39482024-02-13T21:40:56.003Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221445215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39492024-02-13T21:40:56.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.22162564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39502024-02-13T21:40:56.003Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221662636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39512024-02-13T21:40:56.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221699653Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39522024-02-13T21:40:56.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221731951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39532024-02-13T21:40:56.003Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221747886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39542024-02-13T21:40:56.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.221768718Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39552024-02-13T21:40:56.004Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.22192826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39562024-02-13T21:40:56.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222324916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39572024-02-13T21:40:56.004Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222362973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39582024-02-13T21:40:56.004Z{"msg":"[0]R flush_numbers[0..12]: [57, 59, 53, 53, 50, 57, 57, 55, 49, 55, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222380456Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39592024-02-13T21:40:56.004Z{"msg":"[0]R generation[0..12]: [34, 35, 33, 33, 30, 34, 34, 34, 29, 34, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222403278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39602024-02-13T21:40:56.004Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222418033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39612024-02-13T21:40:56.004Z{"msg":"[1]R flush_numbers[0..12]: [57, 59, 53, 53, 50, 57, 57, 55, 49, 55, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222438705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39622024-02-13T21:40:56.005Z{"msg":"[1]R generation[0..12]: [34, 35, 33, 33, 30, 34, 34, 34, 29, 34, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222458838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39632024-02-13T21:40:56.005Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39642024-02-13T21:40:56.005Z,"time":"2024-02-13T21:40:54.222494755Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39652024-02-13T21:40:56.005Z{"msg":"[2]R flush_numbers[0..12]: [57, 59, 53, 53, 50, 57, 57, 55, 49, 55, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222531281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39662024-02-13T21:40:56.005Z{"msg":"[2]R generation[0..12]: [34, 35, 33, 33, 30, 34, 34, 34, 29, 34, 30, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.2225631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39672024-02-13T21:40:56.005Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222585452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39682024-02-13T21:40:56.005Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222605614Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39692024-02-13T21:40:56.006Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222636713Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39702024-02-13T21:40:56.006Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222670921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39712024-02-13T21:40:56.006Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222698121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"":"downstairs","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39722024-02-13T21:40:56.006Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222722152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39732024-02-13T21:40:56.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.22275468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"0","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39742024-02-13T21:40:56.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222790647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"1","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39752024-02-13T21:40:56.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222813019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"client":"2","":"downstairs","session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39762024-02-13T21:40:56.006Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222845238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39772024-02-13T21:40:56.007Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222877376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39782024-02-13T21:40:56.007Z{"msg":"96487f7e-a1ef-4923-bea7-745cfee104ad is now active with session: 7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222902717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39792024-02-13T21:40:56.007Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222926838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282,"session_id":"7e2c9b4d-8d6f-4af3-b9fc-6aceed1bcba9"}
39802024-02-13T21:40:56.007Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:54.222959946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
39812024-02-13T21:40:56.007ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39822024-02-13T21:40:56.007ZLoading write count information from file "/tmp/test_repair_verify.data"
39832024-02-13T21:40:56.007ZRun Repair workload
39842024-02-13T21:40:56.007ZError: Difference in extent metadata found!
39852024-02-13T21:40:56.008Z01/30 Write block 426 len 40960 data: 5 4 5 5 5 4 5 3 4 4
39862024-02-13T21:40:56.008Z02/30 Read block 260 len 16384
39872024-02-13T21:40:56.008Z03/30 Write block 460 len 32768 data: 3 3 3 3 3 3 2 3
39882024-02-13T21:40:56.008Z04/30 Write block 556 len 16384 data: 4 3 3 2
39892024-02-13T21:40:56.008Z05/30 Read block 287 len 8192
39902024-02-13T21:40:56.008Z06/30 Read block 31 len 20480
39912024-02-13T21:40:56.008Z07/30 Write block 221 len 16384 data: 4 4 3 3
39922024-02-13T21:40:56.008Z08/30 Read block 460 len 40960
39932024-02-13T21:40:56.008Z09/30 Read block 107 len 4096
39942024-02-13T21:40:56.008Z10/30 Read block 519 len 28672
39952024-02-13T21:40:56.008Z11/30 Write block 233 len 12288 data: 4 6 6
39962024-02-13T21:40:56.009Z12/30 Write block 266 len 20480 data: 4 4 4 3 3
39972024-02-13T21:40:56.009Z13/30 Write block 375 len 12288 data: 6 4 4
39982024-02-13T21:40:56.009Z14/30 Write block 161 len 16384 data: 3 3 3 2
39992024-02-13T21:40:56.009Z15/30 Read block 569 len 40960
40002024-02-13T21:40:56.009Z16/30 Read block 13 len 36864
40012024-02-13T21:40:56.009Z17/30 Write block 540 len 36864 data: 5 6 4 4 5 6 5 4 3
40022024-02-13T21:40:56.009Z18/30 Read block 196 len 24576
40032024-02-13T21:40:56.009Z19/30 Write block 38 len 20480 data: 4 4 4 3 3
40042024-02-13T21:40:56.009Z20/30 Read block 189 len 28672
40052024-02-13T21:40:56.010Z21/30 Read block 365 len 32768
40062024-02-13T21:40:56.010Z22/30 Read block 447 len 20480
40072024-02-13T21:40:56.010Z23/30 Read block 543 len 20480
40082024-02-13T21:40:56.010Z24/30 Read block 338 len 32768
40092024-02-13T21:40:56.010Z25/30 Read block 31 len 16384
40102024-02-13T21:40:56.010Z26/30 Read block 386 len 36864
40112024-02-13T21:40:56.010Z27/30 Read block 488 len 24576
40122024-02-13T21:40:56.010Z28/30 Write block 355 len 24576 data: 4 5 5 4 4 4
40132024-02-13T21:40:56.010Z29/30 Write block 130 len 12288 data: 2 4 4
40142024-02-13T21:40:56.010Z30/30 Read block 174 len 36864
40152024-02-13T21:40:56.010Z----------------------------------------------------------------
40162024-02-13T21:40:56.010Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40172024-02-13T21:40:56.011ZStates: Active Active Active
40182024-02-13T21:40:56.011ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40192024-02-13T21:40:56.011Z 1 Acked 1000 Write 10 Sent Done Done false
40202024-02-13T21:40:56.018Z 2 Acked 1001 Read 4 Sent Done Done false
40212024-02-13T21:40:56.018Z 3 Acked 1002 Write 8 Sent Done Done false
40222024-02-13T21:40:56.018Z 4 Acked 1003 Write 4 Sent Done Done false
40232024-02-13T21:40:56.018Z 5 Acked 1004 Read 2 Sent Done Done false
40242024-02-13T21:40:56.019Z 6 Acked 1005 Read 5 Sent Done Done false
40252024-02-13T21:40:56.019Z 7 Acked 1006 Write 4 Sent Done Done false
40262024-02-13T21:40:56.019Z 8 Acked 1007 Read 10 Sent Done Done false
40272024-02-13T21:40:56.019Z 9 Acked 1008 Read 1 Sent Done Done false
40282024-02-13T21:40:56.019Z 10 Acked 1009 Read 7 Sent Done Done false
40292024-02-13T21:40:56.019Z 11 Acked 1010 Write 3 Sent Done Done false
40302024-02-13T21:40:56.019Z 12 Acked 1011 Write 5 Sent Done Done false
40312024-02-13T21:40:56.020Z 13 Acked 1012 Write 3 Sent Done Done false
40322024-02-13T21:40:56.020Z 14 Acked 1013 Write 4 Sent Done Done false
40332024-02-13T21:40:56.020Z 15 Acked 1014 Read 10 Sent Done Done false
40342024-02-13T21:40:56.020Z 16 Acked 1015 Read 9 Sent Done Done false
40352024-02-13T21:40:56.020Z 17 Acked 1016 Write 9 Sent Done Done false
40362024-02-13T21:40:56.020Z 18 Acked 1017 Read 6 Sent Done Done false
40372024-02-13T21:40:56.020Z 19 Acked 1018 Write 5 Sent Done Done false
40382024-02-13T21:40:56.021Z 20 Acked 1019 Read 7 Sent Done Done false
40392024-02-13T21:40:56.021Z 21 Acked 1020 Read 8 Sent Done Done false
40402024-02-13T21:40:56.021Z 22 Acked 1021 Read 5 Sent Done Done false
40412024-02-13T21:40:56.021Z 23 Acked 1022 Read 5 Sent Done Done false
40422024-02-13T21:40:56.021Z 24 Acked 1023 Read 8 Sent Done Done false
40432024-02-13T21:40:56.021Z 25 Acked 1024 Read 4 Sent Done Done false
40442024-02-13T21:40:56.022Z 26 Acked 1025 Read 9 Sent Done Done false
40452024-02-13T21:40:56.022Z 27 Acked 1026 Read 6 Sent Done Done false
40462024-02-13T21:40:56.022Z 28 Acked 1027 Write 6 Sent Done Done false
40472024-02-13T21:40:56.022Z 29 Acked 1028 Write 3 Sent Done Done false
40482024-02-13T21:40:56.022Z 30 Acked 1029 Read 9 Sent Done Done false
40492024-02-13T21:40:56.022Z STATES DS:0 DS:1 DS:2 TOTAL
40502024-02-13T21:40:56.022Z New 0 0 0 0
40512024-02-13T21:40:56.023Z Sent 30 0 0 30
40522024-02-13T21:40:56.023Z Done 0 30 30 60
40532024-02-13T21:40:56.023Z Skipped 0 0 0 0
40542024-02-13T21:40:56.023Z Error 0 0 0 0
40552024-02-13T21:40:56.023ZLast Flush: 0 0 0
40562024-02-13T21:40:56.024ZDownstairs last five completed:
40572024-02-13T21:40:56.024ZUpstairs last five completed: 30 29 28 27 26
40582024-02-13T21:40:56.024Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:54.250870882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1282}
40592024-02-13T21:40:56.024ZWrote out file "/tmp/test_repair_verify.data"
40602024-02-13T21:40:56.024Z
40612024-02-13T21:40:56.024ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40622024-02-13T21:40:56.024ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40632024-02-13T21:40:56.024Z 0 000-019 34 34 34 57 57 57 F F F
40642024-02-13T21:40:56.025Z 1 020-039 35 35 35 59 59 59 F T T <---
40652024-02-13T21:40:56.025Z 2 040-059 33 33 33 53 53 53 F T T <---
40662024-02-13T21:40:56.025Z 3 060-079 33 33 33 53 53 53 F F F
40672024-02-13T21:40:56.025Z 4 080-099 30 30 30 50 50 50 F F F
40682024-02-13T21:40:56.025Z 5 100-119 34 34 34 57 57 57 F F F
40692024-02-13T21:40:56.025Z 6 120-139 34 34 34 57 57 57 F T T <---
40702024-02-13T21:40:56.025Z 7 140-159 34 34 34 55 55 55 F F F
40712024-02-13T21:40:56.026Z 8 160-179 29 29 29 49 49 49 F T T <---
40722024-02-13T21:40:56.026Z 9 180-199 34 34 34 55 55 55 F F F
40732024-02-13T21:40:56.026Z 10 200-219 30 30 30 50 50 50 F F F
40742024-02-13T21:40:56.026Z 11 220-239 30 30 30 50 50 50 F T T <---
40752024-02-13T21:40:56.026Z 12 240-259 32 32 32 51 51 51 F F F
40762024-02-13T21:40:56.027Z 13 260-279 34 34 34 58 58 58 F T T <---
40772024-02-13T21:40:56.027Z 14 280-299 33 33 33 53 53 53 F F F
40782024-02-13T21:40:56.027Z 15 300-319 34 34 34 58 58 58 F F F
40792024-02-13T21:40:56.027Z 16 320-339 32 32 32 51 51 51 F F F
40802024-02-13T21:40:56.027Z 17 340-359 32 32 32 51 51 51 F T T <---
40812024-02-13T21:40:56.027Z 18 360-379 34 34 34 58 58 58 F T T <---
40822024-02-13T21:40:56.027Z 19 380-399 34 34 34 55 55 55 F F F
40832024-02-13T21:40:56.028Z 20 400-419 32 32 32 51 51 51 F F F
40842024-02-13T21:40:56.028Z 21 420-439 35 35 35 59 59 59 F T T <---
40852024-02-13T21:40:56.028Z 22 440-459 30 30 30 50 50 50 F F F
40862024-02-13T21:40:56.028Z 23 460-479 30 30 30 50 50 50 F T T <---
40872024-02-13T21:40:56.028Z 24 480-499 33 33 33 53 53 53 F F F
40882024-02-13T21:40:56.028Z 25 500-519 34 34 34 56 56 56 F F F
40892024-02-13T21:40:56.028Z 26 520-539 34 34 34 56 56 56 F F F
40902024-02-13T21:40:56.028Z 27 540-559 30 30 30 50 50 50 F T T <---
40912024-02-13T21:40:56.028Z 28 560-579 26 26 26 46 46 46 F F F
40922024-02-13T21:40:56.029Z 29 580-599 34 34 34 57 57 57 F F F
40932024-02-13T21:40:56.029ZMax gen: 35, Max flush: 59
40942024-02-13T21:40:56.029ZA Difference in extent metadata is expected here
40952024-02-13T21:40:56.029ZOn loop 18
40962024-02-13T21:40:56.029Z
40972024-02-13T21:40:56.029ZVerifying data now
40982024-02-13T21:41:00.039ZNo change after verify
40992024-02-13T21:41:00.466ZLoop: 18 Downstairs dump after verify (and repair):
41002024-02-13T21:41:00.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41012024-02-13T21:41:00.467Z 0 000-019 34 34 34 57 57 57 F F F
41022024-02-13T21:41:00.467Z 1 020-039 36 36 36 60 60 60 F F F
41032024-02-13T21:41:00.467Z 2 040-059 36 36 36 60 60 60 F F F
41042024-02-13T21:41:00.467Z 3 060-079 33 33 33 53 53 53 F F F
41052024-02-13T21:41:00.467Z 4 080-099 30 30 30 50 50 50 F F F
41062024-02-13T21:41:00.467Z 5 100-119 34 34 34 57 57 57 F F F
41072024-02-13T21:41:00.468Z 6 120-139 36 36 36 60 60 60 F F F
41082024-02-13T21:41:00.468Z 7 140-159 34 34 34 55 55 55 F F F
41092024-02-13T21:41:00.468Z 8 160-179 36 36 36 60 60 60 F F F
41102024-02-13T21:41:00.468Z 9 180-199 34 34 34 55 55 55 F F F
41112024-02-13T21:41:00.468Z 10 200-219 30 30 30 50 50 50 F F F
41122024-02-13T21:41:00.468Z 11 220-239 36 36 36 60 60 60 F F F
41132024-02-13T21:41:00.468Z 12 240-259 32 32 32 51 51 51 F F F
41142024-02-13T21:41:00.468Z 13 260-279 36 36 36 60 60 60 F F F
41152024-02-13T21:41:00.469Z 14 280-299 33 33 33 53 53 53 F F F
41162024-02-13T21:41:00.469Z 15 300-319 34 34 34 58 58 58 F F F
41172024-02-13T21:41:00.469Z 16 320-339 32 32 32 51 51 51 F F F
41182024-02-13T21:41:00.469Z 17 340-359 36 36 36 60 60 60 F F F
41192024-02-13T21:41:00.469Z 18 360-379 36 36 36 60 60 60 F F F
41202024-02-13T21:41:00.469Z 19 380-399 34 34 34 55 55 55 F F F
41212024-02-13T21:41:00.469Z 20 400-419 32 32 32 51 51 51 F F F
41222024-02-13T21:41:00.469Z 21 420-439 36 36 36 60 60 60 F F F
41232024-02-13T21:41:00.469Z 22 440-459 30 30 30 50 50 50 F F F
41242024-02-13T21:41:00.469Z 23 460-479 36 36 36 60 60 60 F F F
41252024-02-13T21:41:00.470Z 24 480-499 33 33 33 53 53 53 F F F
41262024-02-13T21:41:00.470Z 25 500-519 34 34 34 56 56 56 F F F
41272024-02-13T21:41:00.470Z 26 520-539 34 34 34 56 56 56 F F F
41282024-02-13T21:41:00.470Z 27 540-559 36 36 36 60 60 60 F F F
41292024-02-13T21:41:00.470Z 28 560-579 26 26 26 46 46 46 F F F
41302024-02-13T21:41:00.470Z 29 580-599 34 34 34 57 57 57 F F F
41312024-02-13T21:41:00.470ZMax gen: 36, Max flush: 60
41322024-02-13T21:41:00.471Z
41332024-02-13T21:41:00.471ZBegin loop 19
41342024-02-13T21:41:00.471ZDownstairs to restart: 2
41352024-02-13T21:41:00.512Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.875573905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
41362024-02-13T21:41:00.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.875831192Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
41372024-02-13T21:41:00.513Z{"msg":"Upstairs <-> Downstairs Message Version: 5Crucible runtime is spawned
41382024-02-13T21:41:00.513Z","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.875934724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
41392024-02-13T21:41:00.513Z{"msg":"Crucible c4361c0a-c2b8-4ad1-8c16-edaa2ffbdb34 has session id: 79b1c19e-5923-4e45-8856-d2c960ea35c2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.875975369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41402024-02-13T21:41:00.513Z{"msg":"Upstairs opts: Upstairs UUID: c4361c0a-c2b8-4ad1-8c16-edaa2ffbdb34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876037077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41412024-02-13T21:41:00.513Z{"msg":"Crucible stats registered with UUID: c4361c0a-c2b8-4ad1-8c16-edaa2ffbdb34","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876079092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41422024-02-13T21:41:00.514Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876117878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
41432024-02-13T21:41:00.514Z{"msg":"c4361c0a-c2b8-4ad1-8c16-edaa2ffbdb34 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876154155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41442024-02-13T21:41:00.514Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876191621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41452024-02-13T21:41:00.514Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876230747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41462024-02-13T21:41:00.514Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876251839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41472024-02-13T21:41:00.515Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876298332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41482024-02-13T21:41:00.515Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876329411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41492024-02-13T21:41:00.515Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.87636079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41502024-02-13T21:41:00.515Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876390629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41512024-02-13T21:41:00.515Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876435443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41522024-02-13T21:41:00.515Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876475848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41532024-02-13T21:41:00.515Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876514524Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41542024-02-13T21:41:00.515Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.876543234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41552024-02-13T21:41:00.516Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.87658149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"io task","client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41562024-02-13T21:41:00.516Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877265992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41572024-02-13T21:41:00.516Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877301039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41582024-02-13T21:41:00.516Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877318102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41592024-02-13T21:41:00.516Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877337915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41602024-02-13T21:41:00.517Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877811194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41612024-02-13T21:41:00.517Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.877944835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41622024-02-13T21:41:00.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878104627Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41632024-02-13T21:41:00.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878140614Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41642024-02-13T21:41:00.517Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878301976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41652024-02-13T21:41:00.517Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878351208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41662024-02-13T21:41:00.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878382027Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41672024-02-13T21:41:00.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878452091Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41682024-02-13T21:41:00.518Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878471864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41692024-02-13T21:41:00.518Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878493556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41702024-02-13T21:41:00.518Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.878694393Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41712024-02-13T21:41:00.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879132704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41722024-02-13T21:41:00.518Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.87917271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41732024-02-13T21:41:00.519Z{"msg":"[0]R flush_numbers[0..12]: [57, 60, 60, 53, 50, 57, 60, 55, 60, 55, 50, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879189274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41742024-02-13T21:41:00.519Z{"msg":"[0]R generation[0..12]: [34, 36, 36, 33, 30, 34, 36, 34, 36, 34, 30, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879211306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41752024-02-13T21:41:00.519Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879270094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41762024-02-13T21:41:00.519Z{"msg":"[1]R flush_numbers[0..12]: [57, 60, 60, 53, 50, 57, 60, 55, 60, 55, 50, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879295655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41772024-02-13T21:41:00.519Z{"msg":"[1]R generation[0..12]: [34, 36, 36, 33, 30, 34, 36, 34, 36, 34, 30, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.87931009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41782024-02-13T21:41:00.519Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879331982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41792024-02-13T21:41:00.519Z{"msg":"[2]R flush_numbers[0..12]: [57, 60, 60, 53, 50, 57, 60, 55, 60, 55, 50, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879346447Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41802024-02-13T21:41:00.520Z{"msg":"[2]R generation[0..12]: [34, 36, 36, 33, 30, 34, 36, 34, 36, 34, 30, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879360741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41812024-02-13T21:41:00.520Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879374956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41822024-02-13T21:41:00.520Z{"msg":"Max found gen is 37","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879400527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41832024-02-13T21:41:00.520Z{"msg":"Generation requested: 38 >= found:37","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
41842024-02-13T21:41:00.520Z2024-02-13T21:40:58.879414832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41852024-02-13T21:41:00.520Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879430866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41862024-02-13T21:41:00.520Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879450379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"":"downstairs","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41872024-02-13T21:41:00.520Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879464114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41882024-02-13T21:41:00.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879477179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"0","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41892024-02-13T21:41:00.520Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879497112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"1","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41902024-02-13T21:41:00.521Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879510417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"client":"2","":"downstairs","session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41912024-02-13T21:41:00.521Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.87953009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41922024-02-13T21:41:00.521Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879542975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41932024-02-13T21:41:00.521Z{"msg":"c4361c0a-c2b8-4ad1-8c16-edaa2ffbdb34 is now active with session: 79b1c19e-5923-4e45-8856-d2c960ea35c2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879562308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41942024-02-13T21:41:00.521Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879575474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290,"session_id":"79b1c19e-5923-4e45-8856-d2c960ea35c2"}
41952024-02-13T21:41:00.521Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:40:58.879587639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
41962024-02-13T21:41:00.521ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41972024-02-13T21:41:00.522ZLoading write count information from file "/tmp/test_repair_verify.data"
41982024-02-13T21:41:00.522ZRun Repair workload
41992024-02-13T21:41:00.522Z01/30 Write block 116 len 32768 data: 4 4 4 4 5 6 6 5
42002024-02-13T21:41:00.522Z02/30 Write block 387 len 40960 data: 5 4 3 3 3 3 3 3 2 2
42012024-02-13T21:41:00.522Z03/30 Write block 98 len 32768 data: 3 3 3 4 5 5 4 3
42022024-02-13T21:41:00.522Z04/30 Read block 332 len 8192
42032024-02-13T21:41:00.522Z05/30 Write block 354 len 24576 data: 4 5 6 6 5 5
42042024-02-13T21:41:00.523Z06/30 Read block 559 len 16384
42052024-02-13T21:41:00.523Z07/30 Flush
42062024-02-13T21:41:00.523Z08/30 Write block 478 len 40960 data: 3 3 3 3 3 3 3 3 3 3
42072024-02-13T21:41:00.523Z09/30 Read block 326 len 16384
42082024-02-13T21:41:00.523Z10/30 Read block 111 len 20480
42092024-02-13T21:41:00.523Z11/30 Read block 333 len 36864
42102024-02-13T21:41:00.523Z12/30 Read block 343 len 24576
42112024-02-13T21:41:00.523Z13/30 Read block 475 len 32768
42122024-02-13T21:41:00.523Z14/30 Read block 61 len 20480
42132024-02-13T21:41:00.524Z15/30 Write block 222 len 32768 data: 5 4 4 3 4 5 5 4
42142024-02-13T21:41:00.524Z16/30 Write block 366 len 40960 data: 4 5 4 4 4 4 3 4 4 7
42152024-02-13T21:41:00.524Z17/30 Write block 496 len 12288 data: 6 6 7
42162024-02-13T21:41:00.524Z18/30 Write block 177 len 8192 data: 3 2
42172024-02-13T21:41:00.524Z19/30 Write block 238 len 20480 data: 7 6 6 6 5
42182024-02-13T21:41:00.524Z20/30 Write block 48 len 40960 data: 4 5 4 3 3 3 4 4 5 5
42192024-02-13T21:41:00.524Z21/30 Read block 531 len 24576
42202024-02-13T21:41:00.524Z22/30 Write block 582 len 16384 data: 3 5 4 5
42212024-02-13T21:41:00.525Z23/30 Read block 211 len 4096
42222024-02-13T21:41:00.525Z24/30 Write block 83 len 28672 data: 5 4 4 3 3 4 3
42232024-02-13T21:41:00.525Z25/30 Write block 554 len 20480 data: 4 4 5 4 4
42242024-02-13T21:41:00.525Z26/30 Read block 174 len 12288
42252024-02-13T21:41:00.525Z27/30 Read block 19 len 4096
42262024-02-13T21:41:00.525Z28/30 Read block 165 len 20480
42272024-02-13T21:41:00.525Z29/30 Read block 132 len 8192
42282024-02-13T21:41:00.525Z30/30 Write block 455 len 24576 data: 3 3 3 3 4 4
42292024-02-13T21:41:00.526Z----------------------------------------------------------------
42302024-02-13T21:41:00.526Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42312024-02-13T21:41:00.526ZStates: Active Active Active
42322024-02-13T21:41:00.526ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42332024-02-13T21:41:00.526Z 1 Acked 1000 Write 8 Done Done Done false
42342024-02-13T21:41:00.526Z 2 Acked 1001 Write 10 Done Done Done false
42352024-02-13T21:41:00.532Z 3 Acked 1002 Write 8 Done Done Sent false
42362024-02-13T21:41:00.532Z 4 Acked 1003 Read 2 Done Done Sent false
42372024-02-13T21:41:00.532Z 5 Acked 1004 Write 6 Done Done Sent false
42382024-02-13T21:41:00.532Z 6 Acked 1005 Read 4 Done Done Sent false
42392024-02-13T21:41:00.532Z 7 Acked 1006 Flush 0 Done Done Sent false
42402024-02-13T21:41:00.532Z 8 Acked 1007 Write 10 Done Done Sent false
42412024-02-13T21:41:00.532Z 9 Acked 1008 Read 4 Done Done Sent false
42422024-02-13T21:41:00.532Z 10 Acked 1009 Read 5 Done Done Sent false
42432024-02-13T21:41:00.533Z 11 Acked 1010 Read 9 Done Done Sent false
42442024-02-13T21:41:00.533Z 12 Acked 1011 Read 6 Done Done Sent false
42452024-02-13T21:41:00.533Z 13 Acked 1012 Read 8 Done Done Sent false
42462024-02-13T21:41:00.533Z 14 Acked 1013 Read 5 Done Done Sent false
42472024-02-13T21:41:00.533Z 15 Acked 1014 Write 8 Done Done Sent false
42482024-02-13T21:41:00.533Z 16 Acked 1015 Write 10 Done Done Sent false
42492024-02-13T21:41:00.533Z 17 Acked 1016 Write 3 Done Done Sent false
42502024-02-13T21:41:00.533Z 18 Acked 1017 Write 2 Done Done Sent false
42512024-02-13T21:41:00.533Z 19 Acked 1018 Write 5 Done Done Sent false
42522024-02-13T21:41:00.533Z 20 Acked 1019 Write 10 Done Done Sent false
42532024-02-13T21:41:00.533Z 21 Acked 1020 Read 6 Done Done Sent false
42542024-02-13T21:41:00.533Z 22 Acked 1021 Write 4 Done Done Sent false
42552024-02-13T21:41:00.533Z 23 Acked 1022 Read 1 Done Done Sent false
42562024-02-13T21:41:00.533Z 24 Acked 1023 Write 7 Done Done Sent false
42572024-02-13T21:41:00.533Z 25 Acked 1024 Write 5 Done Done Sent false
42582024-02-13T21:41:00.534Z 26 Acked 1025 Read 3 Done Done Sent false
42592024-02-13T21:41:00.534Z 27 Acked 1026 Read 1 Done Done Sent false
42602024-02-13T21:41:00.534Z 28 Acked 1027 Read 5 Done Done Sent false
42612024-02-13T21:41:00.534Z 29 Acked 1028 Read 2 Done Done Sent false
42622024-02-13T21:41:00.534Z 30 Acked 1029 Write 6 Sent Sent Sent false
42632024-02-13T21:41:00.534Z STATES DS:0 DS:1 DS:2 TOTAL
42642024-02-13T21:41:00.534Z New 0 0 0 0
42652024-02-13T21:41:00.534Z Sent 1 1 28 30
42662024-02-13T21:41:00.534Z Done 29 29 2 60
42672024-02-13T21:41:00.534Z Skipped 0 0 0 0
42682024-02-13T21:41:00.534Z Error 0 0 0 0
42692024-02-13T21:41:00.534ZLast Flush: 1006 1006 0
42702024-02-13T21:41:00.534ZDownstairs last five completed:
42712024-02-13T21:41:00.534ZUpstairs last five completed: 30 29 28 27 26
42722024-02-13T21:41:00.534Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:40:58.908765515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1290}
42732024-02-13T21:41:00.534ZWrote out file "/tmp/test_repair_verify.data"
42742024-02-13T21:41:00.534Z
42752024-02-13T21:41:00.534ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42762024-02-13T21:41:00.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42772024-02-13T21:41:00.535Z 0 000-019 34 34 34 57 57 57 F F F
42782024-02-13T21:41:00.535Z 1 020-039 36 36 36 60 60 60 F F F
42792024-02-13T21:41:00.535Z 2 040-059 36 36 36 60 60 60 T T F <---
42802024-02-13T21:41:00.535Z 3 060-079 33 33 33 53 53 53 F F F
42812024-02-13T21:41:00.535Z 4 080-099 38 38 30 61 61 50 T T F <---
42822024-02-13T21:41:00.535Z 5 100-119 38 38 34 61 61 57 F F T <---
42832024-02-13T21:41:00.535Z 6 120-139 38 38 36 61 61 60 F F T <---
42842024-02-13T21:41:00.535Z 7 140-159 34 34 34 55 55 55 F F F
42852024-02-13T21:41:00.535Z 8 160-179 36 36 36 60 60 60 T T F <---
42862024-02-13T21:41:00.535Z 9 180-199 34 34 34 55 55 55 F F F
42872024-02-13T21:41:00.535Z 10 200-219 30 30 30 50 50 50 F F F
42882024-02-13T21:41:00.535Z 11 220-239 36 36 36 60 60 60 T T F <---
42892024-02-13T21:41:00.535Z 12 240-259 32 32 32 51 51 51 T T F <---
42902024-02-13T21:41:00.535Z 13 260-279 36 36 36 60 60 60 F F F
42912024-02-13T21:41:00.535Z 14 280-299 33 33 33 53 53 53 F F F
42922024-02-13T21:41:00.535Z 15 300-319 34 34 34 58 58 58 F F F
42932024-02-13T21:41:00.535Z 16 320-339 32 32 32 51 51 51 F F F
42942024-02-13T21:41:00.535Z 17 340-359 38 38 36 61 61 60 F F F <---
42952024-02-13T21:41:00.535Z 18 360-379 36 36 36 60 60 60 T T F <---
42962024-02-13T21:41:00.535Z 19 380-399 38 38 34 61 61 55 F F T <---
42972024-02-13T21:41:00.535Z 20 400-419 32 32 32 51 51 51 F F F
42982024-02-13T21:41:00.536Z 21 420-439 36 36 36 60 60 60 F F F
42992024-02-13T21:41:00.536Z 22 440-459 30 30 30 50 50 50 T T F <---
43002024-02-13T21:41:00.536Z 23 460-479 36 36 36 60 60 60 T T F <---
43012024-02-13T21:41:00.536Z 24 480-499 33 33 33 53 53 53 T T F <---
43022024-02-13T21:41:00.536Z 25 500-519 34 34 34 56 56 56 F F F
43032024-02-13T21:41:00.536Z 26 520-539 34 34 34 56 56 56 F F F
43042024-02-13T21:41:00.536Z 27 540-559 36 36 36 60 60 60 T T F <---
43052024-02-13T21:41:00.536Z 28 560-579 26 26 26 46 46 46 F F F
43062024-02-13T21:41:00.536Z 29 580-599 34 34 34 57 57 57 T T F <---
43072024-02-13T21:41:00.536ZMax gen: 38, Max flush: 61
43082024-02-13T21:41:00.536ZError: Difference in extent metadata found!
43092024-02-13T21:41:00.536ZA Difference in extent metadata is expected here
43102024-02-13T21:41:00.536ZOn loop 19
43112024-02-13T21:41:00.536Z
43122024-02-13T21:41:00.536ZVerifying data now
43132024-02-13T21:41:05.844ZNo change after verify
43142024-02-13T21:41:05.844ZLoop: 19 Downstairs dump after verify (and repair):
43152024-02-13T21:41:05.859ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43162024-02-13T21:41:05.860Z 0 000-019 34 34 34 57 57 57 F F F
43172024-02-13T21:41:05.860Z 1 020-039 36 36 36 60 60 60 F F F
43182024-02-13T21:41:05.860Z 2 040-059 39 39 39 62 62 62 F F F
43192024-02-13T21:41:05.860Z 3 060-079 33 33 33 53 53 53 F F F
43202024-02-13T21:41:05.860Z 4 080-099 39 39 39 62 62 62 F F F
43212024-02-13T21:41:05.860Z 5 100-119 38 38 38 61 61 61 F F F
43222024-02-13T21:41:05.860Z 6 120-139 38 38 38 61 61 61 F F F
43232024-02-13T21:41:05.860Z 7 140-159 34 34 34 55 55 55 F F F
43242024-02-13T21:41:05.861Z 8 160-179 39 39 39 62 62 62 F F F
43252024-02-13T21:41:05.861Z 9 180-199 34 34 34 55 55 55 F F F
43262024-02-13T21:41:05.861Z 10 200-219 30 30 30 50 50 50 F F F
43272024-02-13T21:41:05.870Z 11 220-239 39 39 39 62 62 62 F F F
43282024-02-13T21:41:05.871Z 12 240-259 39 39 39 62 62 62 F F F
43292024-02-13T21:41:05.871Z 13 260-279 36 36 36 60 60 60 F F F
43302024-02-13T21:41:05.871Z 14 280-299 33 33 33 53 53 53 F F F
43312024-02-13T21:41:05.871Z 15 300-319 34 34 34 58 58 58 F F F
43322024-02-13T21:41:05.871Z 16 320-339 32 32 32 51 51 51 F F F
43332024-02-13T21:41:05.871Z 17 340-359 38 38 38 61 61 61 F F F
43342024-02-13T21:41:05.871Z 18 360-379 39 39 39 62 62 62 F F F
43352024-02-13T21:41:05.871Z 19 380-399 38 38 38 61 61 61 F F F
43362024-02-13T21:41:05.872Z 20 400-419 32 32 32 51 51 51 F F F
43372024-02-13T21:41:05.872Z 21 420-439 36 36 36 60 60 60 F F F
43382024-02-13T21:41:05.872Z 22 440-459 39 39 39 62 62 62 F F F
43392024-02-13T21:41:05.872Z 23 460-479 39 39 39 62 62 62 F F F
43402024-02-13T21:41:05.872Z 24 480-499 39 39 39 62 62 62 F F F
43412024-02-13T21:41:05.873Z 25 500-519 34 34 34 56 56 56 F F F
43422024-02-13T21:41:05.873Z 26 520-539 34 34 34 56 56 56 F F F
43432024-02-13T21:41:05.873Z 27 540-559 39 39 39 62 62 62 F F F
43442024-02-13T21:41:05.873Z 28 560-579 26 26 26 46 46 46 F F F
43452024-02-13T21:41:05.873Z 29 580-599 39 39 39 62 62 62 F F F
43462024-02-13T21:41:05.873ZMax gen: 39, Max flush: 62
43472024-02-13T21:41:05.873Z
43482024-02-13T21:41:05.874ZBegin loop 20
43492024-02-13T21:41:05.874ZDownstairs to restart: 2
43502024-02-13T21:41:05.883Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692248787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
43512024-02-13T21:41:05.883Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692564492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
43522024-02-13T21:41:05.883Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692593442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
43532024-02-13T21:41:05.883Z{"msg":"Crucible 21fa7040-1635-4023-b639-4ddc6ebd34a3 has session id: 7c84a7a6-a08e-43f7-a6b2-690f123e1606","v":0,"name":"crucible","level":30Crucible runtime is spawned
43542024-02-13T21:41:05.883Z,"time":"2024-02-13T21:41:04.69262542Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43552024-02-13T21:41:05.883Z{"msg":"Upstairs opts: Upstairs UUID: 21fa7040-1635-4023-b639-4ddc6ebd34a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692656359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43562024-02-13T21:41:05.884Z{"msg":"Crucible stats registered with UUID: 21fa7040-1635-4023-b639-4ddc6ebd34a3","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692671533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43572024-02-13T21:41:05.884Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692700073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
43582024-02-13T21:41:05.884Z{"msg":"21fa7040-1635-4023-b639-4ddc6ebd34a3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692859425Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43592024-02-13T21:41:05.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692888735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43602024-02-13T21:41:05.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692906958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43612024-02-13T21:41:05.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692930689Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43622024-02-13T21:41:05.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692951512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43632024-02-13T21:41:05.884Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.692974763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43642024-02-13T21:41:05.884Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693001394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43652024-02-13T21:41:05.885Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.69303816Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43662024-02-13T21:41:05.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693058683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43672024-02-13T21:41:05.885Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693086992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43682024-02-13T21:41:05.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693187186Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43692024-02-13T21:41:05.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693357154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43702024-02-13T21:41:05.899Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693400949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"io task","client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43712024-02-13T21:41:05.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693904716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43722024-02-13T21:41:05.899Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693941593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43732024-02-13T21:41:05.899Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.693958067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43742024-02-13T21:41:05.899Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.69397501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43752024-02-13T21:41:05.899Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.694542764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43762024-02-13T21:41:05.900Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.694571344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43772024-02-13T21:41:05.900Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.694882681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43782024-02-13T21:41:05.900Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.694928555Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43792024-02-13T21:41:05.900Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695103751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43802024-02-13T21:41:05.900Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695144826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43812024-02-13T21:41:05.900Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695160531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43822024-02-13T21:41:05.900Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695174635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43832024-02-13T21:41:05.900Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695195558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43842024-02-13T21:41:05.900Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695209283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43852024-02-13T21:41:05.900Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695415048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43862024-02-13T21:41:05.901Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695804207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43872024-02-13T21:41:05.901Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695832847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43882024-02-13T21:41:05.901Z{"msg":"[0]R flush_numbers[0..12]: [57, 60, 62, 53, 62, 61, 61, 55, 62, 55, 50, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43892024-02-13T21:41:05.901Z{"msg":"[0]R generation[0..12]: [34, 36, 39, 33, 39, 38, 38, 34, 39, 34, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695866414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43902024-02-13T21:41:05.901Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695890656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43912024-02-13T21:41:05.901Z{"msg":"[1]R flush_numbers[0..12]: [57, 60, 62, 53, 62, 61, 61, 55, 62, 55, 50, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.69590521Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43922024-02-13T21:41:05.901Z{"msg":"[1]R generation[0..12]: [34, 36, 39, 33, 39, 38, 38, 34, 39, 34, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695925563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43932024-02-13T21:41:05.902Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695939808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43942024-02-13T21:41:05.902Z{"msg":"[2]R flush_numbers[0..12]: [57, 60, 62, 53, 62, 61, 61, 55, 62, 55, 50, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.69596018Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43952024-02-13T21:41:05.902Z{"msg":"[2]R generation[0..12]: [34, 36, 39, 33, 39, 38, 38, 34, 39, 34, 30, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695973995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43962024-02-13T21:41:05.902Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.695994438Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43972024-02-13T21:41:05.902Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696007773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
43982024-02-13T21:41:05.902Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43992024-02-13T21:41:05.903Z,"time":"2024-02-13T21:41:04.696027246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44002024-02-13T21:41:05.903Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.69604335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44012024-02-13T21:41:05.903Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696063823Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"":"downstairs","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44022024-02-13T21:41:05.903Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696078267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44032024-02-13T21:41:05.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696097181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"0","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44042024-02-13T21:41:05.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696110986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"1","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44052024-02-13T21:41:05.903Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696130748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"client":"2","":"downstairs","session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44062024-02-13T21:41:05.903Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696144423Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44072024-02-13T21:41:05.904Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696162977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44082024-02-13T21:41:05.904Z{"msg":"21fa7040-1635-4023-b639-4ddc6ebd34a3 is now active with session: 7c84a7a6-a08e-43f7-a6b2-690f123e1606","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696176302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44092024-02-13T21:41:05.904Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696194695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298,"session_id":"7c84a7a6-a08e-43f7-a6b2-690f123e1606"}
44102024-02-13T21:41:05.904Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:04.696207161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
44112024-02-13T21:41:05.904ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44122024-02-13T21:41:05.904ZLoading write count information from file "/tmp/test_repair_verify.data"
44132024-02-13T21:41:05.905ZRun Repair workload
44142024-02-13T21:41:05.905Z01/30 Write block 518 len 24576 data: 7 7 7 7 6 5
44152024-02-13T21:41:05.905Z02/30 Write block 263 len 24576 data: 3 5 5 5 5 5
44162024-02-13T21:41:05.905Z03/30 Write block 577 len 8192 data: 4 4
44172024-02-13T21:41:05.905Z04/30 Write block 75 len 28672 data: 3 4 3 3 4 4 4
44182024-02-13T21:41:05.905Z05/30 Flush
44192024-02-13T21:41:05.905Z06/30 Flush
44202024-02-13T21:41:05.905Z07/30 Write block 254 len 8192 data: 3 3
44212024-02-13T21:41:05.905Z08/30 Write block 519 len 16384 data: 8 8 8 7
44222024-02-13T21:41:05.920Z09/30 Read block 555 len 20480
44232024-02-13T21:41:05.920Z10/30 Read block 390 len 36864
44242024-02-13T21:41:05.920Z11/30 Write block 311 len 36864 data: 5 5 3 3 3 3 3 2 2
44252024-02-13T21:41:05.920Z12/30 Read block 136 len 16384
44262024-02-13T21:41:05.920Z13/30 Read block 291 len 28672
44272024-02-13T21:41:05.920Z14/30 Write block 340 len 28672 data: 5 3 3 3 3 3 3
44282024-02-13T21:41:05.920Z15/30 Read block 94 len 20480
44292024-02-13T21:41:05.920Z16/30 Write block 223 len 16384 data: 5 5 4 5
44302024-02-13T21:41:05.920Z17/30 Write block 14 len 12288 data: 5 5 7
44312024-02-13T21:41:05.921Z18/30 Read block 175 len 28672
44322024-02-13T21:41:05.921Z19/30 Write block 103 len 16384 data: 6 5 4 3
44332024-02-13T21:41:05.921Z20/30 Read block 291 len 24576
44342024-02-13T21:41:05.921Z21/30 Read block 48 len 12288
44352024-02-13T21:41:05.921Z22/30 Read block 248 len 28672
44362024-02-13T21:41:05.921Z23/30 Write block 153 len 24576 data: 2 3 4 4 5 5
44372024-02-13T21:41:05.921Z24/30 Write block 129 len 4096 data: 4
44382024-02-13T21:41:05.921Z25/30 Read block 95 len 16384
44392024-02-13T21:41:05.921Z26/30 Read block 590 len 28672
44402024-02-13T21:41:05.921Z27/30 Write block 131 len 12288 data: 5 5 4
44412024-02-13T21:41:05.921Z28/30 Read block 288 len 8192
44422024-02-13T21:41:05.921Z29/30 Write block 415 len 36864 data: 5 6 7 7 7 6 5 4 4
44432024-02-13T21:41:05.921Z30/30 Write block 10 len 40960 data: 3 5 5 5 6 6 8 8 9 7
44442024-02-13T21:41:05.922Z----------------------------------------------------------------
44452024-02-13T21:41:05.922Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44462024-02-13T21:41:05.922ZStates: Active Active Active
44472024-02-13T21:41:05.922ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44482024-02-13T21:41:05.922Z 1 Acked 1000 Write 6 Done Done Sent false
44492024-02-13T21:41:05.922Z 2 Acked 1001 Write 6 Done Done Sent false
44502024-02-13T21:41:05.922Z 3 Acked 1002 Write 2 Done Done Sent false
44512024-02-13T21:41:05.922Z 4 Acked 1003 Write 7 Done Done Sent false
44522024-02-13T21:41:05.922Z 5 Acked 1004 Flush 0 Done Done Sent false
44532024-02-13T21:41:05.922Z 6 Acked 1005 Flush 0 Done Done Sent false
44542024-02-13T21:41:05.923Z 7 Acked 1006 Write 2 Done Done Sent false
44552024-02-13T21:41:05.923Z 8 Acked 1007 Write 4 Done Done Sent false
44562024-02-13T21:41:05.923Z 9 Acked 1008 Read 5 Done Done Sent false
44572024-02-13T21:41:05.923Z 10 Acked 1009 Read 9 Done Done Sent false
44582024-02-13T21:41:05.923Z 11 Acked 1010 Write 9 Done Done Sent false
44592024-02-13T21:41:05.923Z 12 Acked 1011 Read 4 Done Done Sent false
44602024-02-13T21:41:05.923Z 13 Acked 1012 Read 7 Done Done Sent false
44612024-02-13T21:41:05.923Z 14 Acked 1013 Write 7 Done Done Sent false
44622024-02-13T21:41:05.923Z 15 Acked 1014 Read 5 Done Done Sent false
44632024-02-13T21:41:05.923Z 16 Acked 1015 Write 4 Done Done Sent false
44642024-02-13T21:41:05.923Z 17 Acked 1016 Write 3 Done Done Sent false
44652024-02-13T21:41:05.923Z 18 Acked 1017 Read 7 Done Done Sent false
44662024-02-13T21:41:05.923Z 19 Acked 1018 Write 4 Done Done Sent false
44672024-02-13T21:41:05.924Z 20 Acked 1019 Read 6 Done Done Sent false
44682024-02-13T21:41:05.924Z 21 Acked 1020 Read 3 Done Done Sent false
44692024-02-13T21:41:05.924Z 22 Acked 1021 Read 7 Done Done Sent false
44702024-02-13T21:41:05.924Z 23 Acked 1022 Write 6 Done Done Sent false
44712024-02-13T21:41:05.925Z 24 Acked 1023 Write 1 Done Done Sent false
44722024-02-13T21:41:05.925Z 25 Acked 1024 Read 4 Done Done Sent false
44732024-02-13T21:41:05.925Z 26 Acked 1025 Read 7 Done Done Sent false
44742024-02-13T21:41:05.925Z 27 Acked 1026 Write 3 Done Done Sent false
44752024-02-13T21:41:05.926Z 28 Acked 1027 Read 2 Done Done Sent false
44762024-02-13T21:41:05.926Z 29 Acked 1028 Write 9 Done Done Sent false
44772024-02-13T21:41:05.926Z 30 Acked 1029 Write 10 Sent Sent Sent false
44782024-02-13T21:41:05.926Z STATES DS:0 DS:1 DS:2 TOTAL
44792024-02-13T21:41:05.926Z New 0 0 0 0
44802024-02-13T21:41:05.927Z Sent 1 1 30 32
44812024-02-13T21:41:05.927Z Done 29 29 0 58
44822024-02-13T21:41:05.927Z Skipped 0 0 0 0
44832024-02-13T21:41:05.927Z Error 0 0 0 0
44842024-02-13T21:41:05.928ZLast Flush: 1005 1005 0
44852024-02-13T21:41:05.928ZDownstairs last five completed:
44862024-02-13T21:41:05.928ZUpstairs last five completed: 30 29 28 27 26
44872024-02-13T21:41:05.928Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:04.723775291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1298}
44882024-02-13T21:41:05.928ZWrote out file "/tmp/test_repair_verify.data"
44892024-02-13T21:41:06.353Z
44902024-02-13T21:41:06.353ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44912024-02-13T21:41:06.353ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44922024-02-13T21:41:06.353Z 0 000-019 34 34 34 57 57 57 T T F <---
44932024-02-13T21:41:06.353Z 1 020-039 36 36 36 60 60 60 F F F
44942024-02-13T21:41:06.353Z 2 040-059 39 39 39 62 62 62 F F F
44952024-02-13T21:41:06.353Z 3 060-079 40 40 33 63 63 53 F F F <---
44962024-02-13T21:41:06.353Z 4 080-099 40 40 39 63 63 62 F F F <---
44972024-02-13T21:41:06.353Z 5 100-119 38 38 38 61 61 61 T T F <---
44982024-02-13T21:41:06.353Z 6 120-139 38 38 38 61 61 61 T T F <---
44992024-02-13T21:41:06.353Z 7 140-159 34 34 34 55 55 55 T T F <---
45002024-02-13T21:41:06.353Z 8 160-179 39 39 39 62 62 62 F F F
45012024-02-13T21:41:06.353Z 9 180-199 34 34 34 55 55 55 F F F
45022024-02-13T21:41:06.354Z 10 200-219 30 30 30 50 50 50 F F F
45032024-02-13T21:41:06.354Z 11 220-239 39 39 39 62 62 62 T T F <---
45042024-02-13T21:41:06.354Z 12 240-259 39 39 39 62 62 62 T T F <---
45052024-02-13T21:41:06.354Z 13 260-279 40 40 36 63 63 60 F F F <---
45062024-02-13T21:41:06.354Z 14 280-299 33 33 33 53 53 53 F F F
45072024-02-13T21:41:06.354Z 15 300-319 34 34 34 58 58 58 T T F <---
45082024-02-13T21:41:06.354Z 16 320-339 32 32 32 51 51 51 F F F
45092024-02-13T21:41:06.354Z 17 340-359 38 38 38 61 61 61 T T F <---
45102024-02-13T21:41:06.354Z 18 360-379 39 39 39 62 62 62 F F F
45112024-02-13T21:41:06.354Z 19 380-399 38 38 38 61 61 61 F F F
45122024-02-13T21:41:06.354Z 20 400-419 32 32 32 51 51 51 T T F <---
45132024-02-13T21:41:06.354Z 21 420-439 36 36 36 60 60 60 T T F <---
45142024-02-13T21:41:06.354Z 22 440-459 39 39 39 62 62 62 F F F
45152024-02-13T21:41:06.354Z 23 460-479 39 39 39 62 62 62 F F F
45162024-02-13T21:41:06.354Z 24 480-499 39 39 39 62 62 62 F F F
45172024-02-13T21:41:06.354Z 25 500-519 40 40 34 63 63 56 T T F <---
45182024-02-13T21:41:06.354Z 26 520-539 40 40 34 63 63 56 T T F <---
45192024-02-13T21:41:06.354Z 27 540-559 39 39 39 62 62 62 F F F
45202024-02-13T21:41:06.354Z 28 560-579 40 40 26 63 63 46 F F F <---
45212024-02-13T21:41:06.354Z 29 580-599 39 39 39 62 62 62 F F F
45222024-02-13T21:41:06.354ZMax gen: 40, Max flush: 63
45232024-02-13T21:41:06.354ZError: Difference in extent metadata found!
45242024-02-13T21:41:06.354ZA Difference in extent metadata is expected here
45252024-02-13T21:41:06.355ZOn loop 20
45262024-02-13T21:41:06.355Z
45272024-02-13T21:41:06.355ZVerifying data now
45282024-02-13T21:41:10.709ZNo change after verify
45292024-02-13T21:41:11.131ZLoop: 20 Downstairs dump after verify (and repair):
45302024-02-13T21:41:11.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45312024-02-13T21:41:11.156Z 0 000-019 41 41 41 64 64 64 F F F
45322024-02-13T21:41:11.156Z 1 020-039 36 36 36 60 60 60 F F F
45332024-02-13T21:41:11.156Z 2 040-059 39 39 39 62 62 62 F F F
45342024-02-13T21:41:11.156Z 3 060-079 40 40 40 63 63 63 F F F
45352024-02-13T21:41:11.156Z 4 080-099 40 40 40 63 63 63 F F F
45362024-02-13T21:41:11.156Z 5 100-119 41 41 41 64 64 64 F F F
45372024-02-13T21:41:11.156Z 6 120-139 41 41 41 64 64 64 F F F
45382024-02-13T21:41:11.156Z 7 140-159 41 41 41 64 64 64 F F F
45392024-02-13T21:41:11.156Z 8 160-179 39 39 39 62 62 62 F F F
45402024-02-13T21:41:11.157Z 9 180-199 34 34 34 55 55 55 F F F
45412024-02-13T21:41:11.157Z 10 200-219 30 30 30 50 50 50 F F F
45422024-02-13T21:41:11.157Z 11 220-239 41 41 41 64 64 64 F F F
45432024-02-13T21:41:11.157Z 12 240-259 41 41 41 64 64 64 F F F
45442024-02-13T21:41:11.157Z 13 260-279 40 40 40 63 63 63 F F F
45452024-02-13T21:41:11.157Z 14 280-299 33 33 33 53 53 53 F F F
45462024-02-13T21:41:11.157Z 15 300-319 41 41 41 64 64 64 F F F
45472024-02-13T21:41:11.157Z 16 320-339 32 32 32 51 51 51 F F F
45482024-02-13T21:41:11.157Z 17 340-359 41 41 41 64 64 64 F F F
45492024-02-13T21:41:11.157Z 18 360-379 39 39 39 62 62 62 F F F
45502024-02-13T21:41:11.157Z 19 380-399 38 38 38 61 61 61 F F F
45512024-02-13T21:41:11.157Z 20 400-419 41 41 41 64 64 64 F F F
45522024-02-13T21:41:11.158Z 21 420-439 41 41 41 64 64 64 F F F
45532024-02-13T21:41:11.158Z 22 440-459 39 39 39 62 62 62 F F F
45542024-02-13T21:41:11.158Z 23 460-479 39 39 39 62 62 62 F F F
45552024-02-13T21:41:11.158Z 24 480-499 39 39 39 62 62 62 F F F
45562024-02-13T21:41:11.158Z 25 500-519 41 41 41 64 64 64 F F F
45572024-02-13T21:41:11.158Z 26 520-539 41 41 41 64 64 64 F F F
45582024-02-13T21:41:11.158Z 27 540-559 39 39 39 62 62 62 F F F
45592024-02-13T21:41:11.158Z 28 560-579 40 40 40 63 63 63 F F F
45602024-02-13T21:41:11.158Z 29 580-599 39 39 39 62 62 62 F F F
45612024-02-13T21:41:11.158ZMax gen: 41, Max flush: 64
45622024-02-13T21:41:11.158Z
45632024-02-13T21:41:11.158ZBegin loop 21
45642024-02-13T21:41:11.158ZDownstairs to restart: 0
45652024-02-13T21:41:11.158Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552472489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
45662024-02-13T21:41:11.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552775619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
45672024-02-13T21:41:11.159ZCrucible runtime is spawned
45682024-02-13T21:41:11.159Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552819833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
45692024-02-13T21:41:11.159Z{"msg":"Crucible 3ac373c3-46f7-4dd2-bfba-cee3c156fd44 has session id: 8b134afd-bbe9-4ef2-9324-93d3f5e1952a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.5528554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45702024-02-13T21:41:11.159Z{"msg":"Upstairs opts: Upstairs UUID: 3ac373c3-46f7-4dd2-bfba-cee3c156fd44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552898735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45712024-02-13T21:41:11.159Z{"msg":"Crucible stats registered with UUID: 3ac373c3-46f7-4dd2-bfba-cee3c156fd44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552926774Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45722024-02-13T21:41:11.159Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552944428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
45732024-02-13T21:41:11.159Z{"msg":"3ac373c3-46f7-4dd2-bfba-cee3c156fd44 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552957563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45742024-02-13T21:41:11.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.552989562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45752024-02-13T21:41:11.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553017161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45762024-02-13T21:41:11.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553056077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45772024-02-13T21:41:11.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553082638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45782024-02-13T21:41:11.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553117855Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45792024-02-13T21:41:11.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553141546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45802024-02-13T21:41:11.159Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553177383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45812024-02-13T21:41:11.160Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553207892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45822024-02-13T21:41:11.160Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55324305Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45832024-02-13T21:41:11.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55326982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45842024-02-13T21:41:11.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553303178Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45852024-02-13T21:41:11.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.553458531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"io task","client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45862024-02-13T21:41:11.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554047048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45872024-02-13T21:41:11.160Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554092791Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45882024-02-13T21:41:11.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554125199Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45892024-02-13T21:41:11.160Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554155568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45902024-02-13T21:41:11.160Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554654727Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45912024-02-13T21:41:11.160Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.554765937Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45922024-02-13T21:41:11.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555032051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45932024-02-13T21:41:11.160Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555075185Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45942024-02-13T21:41:11.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555185925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45952024-02-13T21:41:11.161Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555218273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45962024-02-13T21:41:11.161Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555233748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45972024-02-13T21:41:11.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55525285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45982024-02-13T21:41:11.161Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555275522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
45992024-02-13T21:41:11.161Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555311769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46002024-02-13T21:41:11.161Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555511387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46012024-02-13T21:41:11.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55591716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46022024-02-13T21:41:11.161Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.555962623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46032024-02-13T21:41:11.161Z{"msg":"[0]R flush_numbers[0..12]: [64, 60, 62, 63, 63, 64, 64, 64, 62, 55, 50, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556022721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46042024-02-13T21:41:11.161Z{"msg":"[0]R generation[0..12]: [41, 36, 39, 40, 40, 41, 41, 41, 39, 34, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556045043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46052024-02-13T21:41:11.161Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556068905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46062024-02-13T21:41:11.161Z{"msg":"[1]R flush_numbers[0..12]: [64, 60, 62, 63, 63, 64, 64, 64, 62, 55, 50, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556083549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46072024-02-13T21:41:11.162Z{"msg":"[1]R generation[0..12]: [41, 36, 39, 40, 40, 41, 41, 41, 39, 34, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556097374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46082024-02-13T21:41:11.162ZWait for a query_work_queue command to finish before sending IO
46092024-02-13T21:41:11.162Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556120806Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46102024-02-13T21:41:11.162Z{"msg":"[2]R flush_numbers[0..12]: [64, 60, 62, 63, 63, 64, 64, 64, 62, 55, 50, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55613566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46112024-02-13T21:41:11.162Z{"msg":"[2]R generation[0..12]: [41, 36, 39, 40, 40, 41, 41, 41, 39, 34, 30, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556150275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46122024-02-13T21:41:11.162Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55616451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46132024-02-13T21:41:11.162Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556189731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46142024-02-13T21:41:11.162Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556203916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46152024-02-13T21:41:11.162Z{"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.55621788Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46162024-02-13T21:41:11.162Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556238223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"":"downstairs","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46172024-02-13T21:41:11.162Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556252218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46182024-02-13T21:41:11.162Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556272071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"0","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46192024-02-13T21:41:11.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556285886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"1","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46202024-02-13T21:41:11.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556305839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"client":"2","":"downstairs","session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46212024-02-13T21:41:11.163Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556319354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46222024-02-13T21:41:11.163Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556337977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46232024-02-13T21:41:11.163Z{"msg":"3ac373c3-46f7-4dd2-bfba-cee3c156fd44 is now active with session: 8b134afd-bbe9-4ef2-9324-93d3f5e1952a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556351242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46242024-02-13T21:41:11.163Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556363338Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306,"session_id":"8b134afd-bbe9-4ef2-9324-93d3f5e1952a"}
46252024-02-13T21:41:11.163Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:09.556381961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
46262024-02-13T21:41:11.163ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46272024-02-13T21:41:11.163ZLoading write count information from file "/tmp/test_repair_verify.data"
46282024-02-13T21:41:11.163ZRun Repair workload
46292024-02-13T21:41:11.163Z01/30 Write block 289 len 40960 data: 4 4 4 4 5 4 4 4 5 4
46302024-02-13T21:41:11.183Z02/30 Read block 115 len 12288
46312024-02-13T21:41:11.183Z03/30 Read block 283 len 32768
46322024-02-13T21:41:11.183Z04/30 Write block 102 len 4096 data: 6
46332024-02-13T21:41:11.183Z05/30 Write block 203 len 12288 data: 4 4 4
46342024-02-13T21:41:11.183Z06/30 Write block 323 len 4096 data: 3
46352024-02-13T21:41:11.183Z07/30 Read block 72 len 28672
46362024-02-13T21:41:11.183Z08/30 Read block 98 len 12288
46372024-02-13T21:41:11.183Z09/30 Write block 341 len 12288 data: 4 4 4
46382024-02-13T21:41:11.183Z10/30 Write block 0 len 32768 data: 3 3 3 3 3 4 4 4
46392024-02-13T21:41:11.183Z11/30 Write block 251 len 28672 data: 3 3 3 4 4 5 5
46402024-02-13T21:41:11.184Z12/30 Write block 473 len 40960 data: 6 6 6 3 3 4 4 4 4 4
46412024-02-13T21:41:11.184Z13/30 Write block 138 len 20480 data: 4 2 3 4 5
46422024-02-13T21:41:11.184Z14/30 Read block 589 len 4096
46432024-02-13T21:41:11.184Z15/30 Write block 142 len 16384 data: 6 5 5 5
46442024-02-13T21:41:11.184Z16/30 Read block 184 len 16384
46452024-02-13T21:41:11.184Z17/30 Write block 520 len 16384 data: 9 9 8 6
46462024-02-13T21:41:11.184Z18/30 Read block 167 len 40960
46472024-02-13T21:41:11.184Z19/30 Write block 9 len 32768 data: 4 4 6 6 6 7 7 9
46482024-02-13T21:41:11.184Z20/30 Read block 435 len 8192
46492024-02-13T21:41:11.184Z21/30 Read block 162 len 16384
46502024-02-13T21:41:11.185Z22/30 Flush
46512024-02-13T21:41:11.185Z23/30 Write block 545 len 40960 data: 7 6 5 4 6 6 7 5 5 5
46522024-02-13T21:41:11.185Z24/30 Read block 256 len 20480
46532024-02-13T21:41:11.185Z25/30 Write block 518 len 12288 data: 8 9 10
46542024-02-13T21:41:11.185Z26/30 Read block 408 len 36864
46552024-02-13T21:41:11.185Z27/30 Read block 214 len 16384
46562024-02-13T21:41:11.185Z28/30 Write block 464 len 36864 data: 4 4 3 4 3 5 5 6 6
46572024-02-13T21:41:11.185Z29/30 Write block 102 len 16384 data: 7 7 6 5
46582024-02-13T21:41:11.185Z30/30 Read block 46 len 8192
46592024-02-13T21:41:11.185Z----------------------------------------------------------------
46602024-02-13T21:41:11.185Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46612024-02-13T21:41:11.185ZStates: Active Active Active
46622024-02-13T21:41:11.186ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46632024-02-13T21:41:11.186Z 1 Acked 1000 Write 10 Done Done Done false
46642024-02-13T21:41:11.186Z 2 Acked 1001 Read 3 Done Done Done false
46652024-02-13T21:41:11.186Z 3 Acked 1002 Read 8 Done Done Done false
46662024-02-13T21:41:11.186Z 4 Acked 1003 Write 1 Done Done Done false
46672024-02-13T21:41:11.186Z 5 Acked 1004 Write 3 Done Done Done false
46682024-02-13T21:41:11.186Z 6 Acked 1005 Write 1 Done Done Done false
46692024-02-13T21:41:11.186Z 7 Acked 1006 Read 7 Sent Done Done false
46702024-02-13T21:41:11.186Z 8 Acked 1007 Read 3 Sent Done Done false
46712024-02-13T21:41:11.186Z 9 Acked 1008 Write 3 Sent Done Done false
46722024-02-13T21:41:11.186Z 10 Acked 1009 Write 8 Sent Done Done false
46732024-02-13T21:41:11.186Z 11 Acked 1010 Write 7 Sent Done Done false
46742024-02-13T21:41:11.187Z 12 Acked 1011 Write 10 Sent Done Done false
46752024-02-13T21:41:11.187Z 13 Acked 1012 Write 5 Sent Done Done false
46762024-02-13T21:41:11.187Z 14 Acked 1013 Read 1 Sent Done Done false
46772024-02-13T21:41:11.187Z 15 Acked 1014 Write 4 Sent Done Done false
46782024-02-13T21:41:11.187Z 16 Acked 1015 Read 4 Sent Done Done false
46792024-02-13T21:41:11.187Z 17 Acked 1016 Write 4 Sent Done Done false
46802024-02-13T21:41:11.187Z 18 Acked 1017 Read 10 Sent Done Done false
46812024-02-13T21:41:11.187Z 19 Acked 1018 Write 8 Sent Done Done false
46822024-02-13T21:41:11.188Z 20 Acked 1019 Read 2 Sent Done Done false
46832024-02-13T21:41:11.188Z 21 Acked 1020 Read 4 Sent Done Done false
46842024-02-13T21:41:11.188Z 22 Acked 1021 Flush 0 Sent Done Done false
46852024-02-13T21:41:11.188Z 23 Acked 1022 Write 10 Sent Done Done false
46862024-02-13T21:41:11.188Z 24 Acked 1023 Read 5 Sent Done Done false
46872024-02-13T21:41:11.188Z 25 Acked 1024 Write 3 Sent Done Done false
46882024-02-13T21:41:11.188Z 26 Acked 1025 Read 9 Sent Done Done false
46892024-02-13T21:41:11.188Z 27 Acked 1026 Read 4 Sent Done Done false
46902024-02-13T21:41:11.188Z 28 Acked 1027 Write 9 Sent Done Done false
46912024-02-13T21:41:11.188Z 29 Acked 1028 Write 4 Sent Done Done false
46922024-02-13T21:41:11.188Z 30 Acked 1029 Read 2 Sent Sent Done false
46932024-02-13T21:41:11.188Z STATES DS:0 DS:1 DS:2 TOTAL
46942024-02-13T21:41:11.189Z New 0 0 0 0
46952024-02-13T21:41:11.189Z Sent 24 1 0 25
46962024-02-13T21:41:11.189Z Done 6 29 30 65
46972024-02-13T21:41:11.189Z Skipped 0 0 0 0
46982024-02-13T21:41:11.189Z Error 0 0 0 0
46992024-02-13T21:41:11.189ZLast Flush: 0 1021 1021
47002024-02-13T21:41:11.189ZDownstairs last five completed:
47012024-02-13T21:41:11.189ZUpstairs last five completed: 30 29 28 27 26
47022024-02-13T21:41:11.189Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:09.587196343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1306}
47032024-02-13T21:41:11.189ZWrote out file "/tmp/test_repair_verify.data"
47042024-02-13T21:41:11.189Z
47052024-02-13T21:41:11.189ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
47062024-02-13T21:41:11.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47072024-02-13T21:41:11.190Z 0 000-019 41 42 42 64 65 65 F F F <---
47082024-02-13T21:41:11.190Z 1 020-039 36 36 36 60 60 60 F F F
47092024-02-13T21:41:11.190Z 2 040-059 39 39 39 62 62 62 F F F
47102024-02-13T21:41:11.190Z 3 060-079 40 40 40 63 63 63 F F F
47112024-02-13T21:41:11.190Z 4 080-099 40 40 40 63 63 63 F F F
47122024-02-13T21:41:11.190Z 5 100-119 41 42 42 64 65 65 T T T <---
47132024-02-13T21:41:11.190Z 6 120-139 41 42 42 64 65 65 F F F <---
47142024-02-13T21:41:11.190Z 7 140-159 41 42 42 64 65 65 F F F <---
47152024-02-13T21:41:11.191Z 8 160-179 39 39 39 62 62 62 F F F
47162024-02-13T21:41:11.191Z 9 180-199 34 34 34 55 55 55 F F F
47172024-02-13T21:41:11.191Z 10 200-219 30 42 42 50 65 65 T F F <---
47182024-02-13T21:41:11.191Z 11 220-239 41 41 41 64 64 64 F F F
47192024-02-13T21:41:11.191Z 12 240-259 41 42 42 64 65 65 F F F <---
47202024-02-13T21:41:11.191Z 13 260-279 40 40 40 63 63 63 F F F
47212024-02-13T21:41:11.191Z 14 280-299 33 42 42 53 65 65 T F F <---
47222024-02-13T21:41:11.191Z 15 300-319 41 41 41 64 64 64 F F F
47232024-02-13T21:41:11.191Z 16 320-339 32 42 42 51 65 65 T F F <---
47242024-02-13T21:41:11.191Z 17 340-359 41 42 42 64 65 65 F F F <---
47252024-02-13T21:41:11.191Z 18 360-379 39 39 39 62 62 62 F F F
47262024-02-13T21:41:11.191Z 19 380-399 38 38 38 61 61 61 F F F
47272024-02-13T21:41:11.192Z 20 400-419 41 41 41 64 64 64 F F F
47282024-02-13T21:41:11.192Z 21 420-439 41 41 41 64 64 64 F F F
47292024-02-13T21:41:11.192Z 22 440-459 39 39 39 62 62 62 F F F
47302024-02-13T21:41:11.208Z 23 460-479 39 42 42 62 65 65 F T T <---
47312024-02-13T21:41:11.208Z 24 480-499 39 42 42 62 65 65 F F F <---
47322024-02-13T21:41:11.209Z 25 500-519 41 41 41 64 64 64 F T T <---
47332024-02-13T21:41:11.209Z 26 520-539 41 42 42 64 65 65 F T T <---
47342024-02-13T21:41:11.209Z 27 540-559 39 39 39 62 62 62 F T T <---
47352024-02-13T21:41:11.210Z 28 560-579 40 40 40 63 63 63 F F F
47362024-02-13T21:41:11.210Z 29 580-599 39 39 39 62 62 62 F F F
47372024-02-13T21:41:11.210ZMax gen: 42, Max flush: 65
47382024-02-13T21:41:11.210ZError: Difference in extent metadata found!
47392024-02-13T21:41:11.210ZA Difference in extent metadata is expected here
47402024-02-13T21:41:11.211ZOn loop 21
47412024-02-13T21:41:11.211Z
47422024-02-13T21:41:11.211ZVerifying data now
47432024-02-13T21:41:16.467ZNo change after verify
47442024-02-13T21:41:16.467ZLoop: 21 Downstairs dump after verify (and repair):
47452024-02-13T21:41:16.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47462024-02-13T21:41:16.482Z 0 000-019 42 42 42 65 65 65 F F F
47472024-02-13T21:41:16.482Z 1 020-039 36 36 36 60 60 60 F F F
47482024-02-13T21:41:16.482Z 2 040-059 39 39 39 62 62 62 F F F
47492024-02-13T21:41:16.482Z 3 060-079 40 40 40 63 63 63 F F F
47502024-02-13T21:41:16.482Z 4 080-099 40 40 40 63 63 63 F F F
47512024-02-13T21:41:16.485Z 5 100-119 43 43 43 66 66 66 F F F
47522024-02-13T21:41:16.485Z 6 120-139 42 42 42 65 65 65 F F F
47532024-02-13T21:41:16.485Z 7 140-159 42 42 42 65 65 65 F F F
47542024-02-13T21:41:16.485Z 8 160-179 39 39 39 62 62 62 F F F
47552024-02-13T21:41:16.486Z 9 180-199 34 34 34 55 55 55 F F F
47562024-02-13T21:41:16.486Z 10 200-219 42 42 42 65 65 65 F F F
47572024-02-13T21:41:16.486Z 11 220-239 41 41 41 64 64 64 F F F
47582024-02-13T21:41:16.486Z 12 240-259 42 42 42 65 65 65 F F F
47592024-02-13T21:41:16.486Z 13 260-279 40 40 40 63 63 63 F F F
47602024-02-13T21:41:16.486Z 14 280-299 42 42 42 65 65 65 F F F
47612024-02-13T21:41:16.486Z 15 300-319 41 41 41 64 64 64 F F F
47622024-02-13T21:41:16.486Z 16 320-339 42 42 42 65 65 65 F F F
47632024-02-13T21:41:16.486Z 17 340-359 42 42 42 65 65 65 F F F
47642024-02-13T21:41:16.486Z 18 360-379 39 39 39 62 62 62 F F F
47652024-02-13T21:41:16.486Z 19 380-399 38 38 38 61 61 61 F F F
47662024-02-13T21:41:16.486Z 20 400-419 41 41 41 64 64 64 F F F
47672024-02-13T21:41:16.486Z 21 420-439 41 41 41 64 64 64 F F F
47682024-02-13T21:41:16.486Z 22 440-459 39 39 39 62 62 62 F F F
47692024-02-13T21:41:16.486Z 23 460-479 43 43 43 66 66 66 F F F
47702024-02-13T21:41:16.486Z 24 480-499 42 42 42 65 65 65 F F F
47712024-02-13T21:41:16.486Z 25 500-519 43 43 43 66 66 66 F F F
47722024-02-13T21:41:16.486Z 26 520-539 43 43 43 66 66 66 F F F
47732024-02-13T21:41:16.486Z 27 540-559 43 43 43 66 66 66 F F F
47742024-02-13T21:41:16.486Z 28 560-579 40 40 40 63 63 63 F F F
47752024-02-13T21:41:16.486Z 29 580-599 39 39 39 62 62 62 F F F
47762024-02-13T21:41:16.486ZMax gen: 43, Max flush: 66
47772024-02-13T21:41:16.486Z
47782024-02-13T21:41:16.486ZBegin loop 22
47792024-02-13T21:41:16.487ZDownstairs to restart: 2
47802024-02-13T21:41:16.504Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
47812024-02-13T21:41:16.504Z,"time":"2024-02-13T21:41:15.313499547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
47822024-02-13T21:41:16.509Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.313899182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
47832024-02-13T21:41:16.509Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.313940707Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
47842024-02-13T21:41:16.509Z{"msg":"Crucible 7ee92567-bfc5-4f62-8d63-5f1794dd6311 has session id: 06c6ef0b-a452-4e87-9215-8d2e4dfc62fb","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.313965468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47852024-02-13T21:41:16.509Z{"msg":"Upstairs opts: Upstairs UUID: 7ee92567-bfc5-4f62-8d63-5f1794dd6311, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314010832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47862024-02-13T21:41:16.509Z{"msg":"Crucible stats registered with UUID: 7ee92567-bfc5-4f62-8d63-5f1794dd6311","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314036492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47872024-02-13T21:41:16.509Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314068821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
47882024-02-13T21:41:16.509Z{"msg":"7ee92567-bfc5-4f62-8d63-5f1794dd6311 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314108696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47892024-02-13T21:41:16.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314146472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47902024-02-13T21:41:16.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314184279Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47912024-02-13T21:41:16.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314210229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47922024-02-13T21:41:16.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314257142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47932024-02-13T21:41:16.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314293629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47942024-02-13T21:41:16.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314309633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47952024-02-13T21:41:16.509Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314323838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47962024-02-13T21:41:16.509Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314378408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47972024-02-13T21:41:16.510Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314408568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47982024-02-13T21:41:16.510Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314436058Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
47992024-02-13T21:41:16.510Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314465327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48002024-02-13T21:41:16.510Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.314493117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"io task","client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48012024-02-13T21:41:16.510Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315162794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48022024-02-13T21:41:16.510Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315217564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48032024-02-13T21:41:16.510Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315286379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48042024-02-13T21:41:16.510Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315318427Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48052024-02-13T21:41:16.510Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315734916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48062024-02-13T21:41:16.510Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.315922798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48072024-02-13T21:41:16.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316437741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48082024-02-13T21:41:16.510Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316472988Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48092024-02-13T21:41:16.510Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316521531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48102024-02-13T21:41:16.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316561427Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48112024-02-13T21:41:16.510Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316588037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48122024-02-13T21:41:16.510Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316627213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48132024-02-13T21:41:16.510Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316685172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48142024-02-13T21:41:16.510Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.316722348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48152024-02-13T21:41:16.515Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317204343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48162024-02-13T21:41:16.515Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317641935Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48172024-02-13T21:41:16.515Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317690187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48182024-02-13T21:41:16.515Z{"msg":"[0]R flush_numbers[0..12]: [65, 60, 62, 63, 63, 66, 65, 65, 62, 55, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317726704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48192024-02-13T21:41:16.515Z{"msg":"[0]R generation[0..12]: [42, 36, 39, 40, 40, 43, 42, 42, 39, 34, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317749756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48202024-02-13T21:41:16.515Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48212024-02-13T21:41:16.515Z{"msg":"[1]R flush_numbers[0..12]: [65, 60, 62, 63, 63, 66, 65, 65, 62, 55, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317786162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48222024-02-13T21:41:16.515Z{"msg":"[1]R generation[0..12]: [42, 36, 39, 40, 40, 43, 42, 42, 39, 34, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317800997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48232024-02-13T21:41:16.515Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317823109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48242024-02-13T21:41:16.515Z{"msg":"[2]R flush_numbers[0..12]: [65, 60, 62, 63, 63, 66, 65, 65, 62, 55, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317837724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48252024-02-13T21:41:16.515Z{"msg":"[2]R generation[0..12]: [42, 36, 39, 40, 40, 43, 42, 42, 39, 34, 42, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317863574Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48262024-02-13T21:41:16.516Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317878709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48272024-02-13T21:41:16.516ZWait for a query_work_queue command to finish before sending IO
48282024-02-13T21:41:16.516Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317899211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48292024-02-13T21:41:16.516Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317923892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48302024-02-13T21:41:16.516Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.317948773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48312024-02-13T21:41:16.516Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.31798414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"":"downstairs","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48322024-02-13T21:41:16.516Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318028844Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48332024-02-13T21:41:16.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318112034Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"0","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48342024-02-13T21:41:16.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318140704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"1","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48352024-02-13T21:41:16.516Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318155578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"client":"2","":"downstairs","session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48362024-02-13T21:41:16.516Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318169363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48372024-02-13T21:41:16.516Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318182219Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48382024-02-13T21:41:16.516Z{"msg":"7ee92567-bfc5-4f62-8d63-5f1794dd6311 is now active with session: 06c6ef0b-a452-4e87-9215-8d2e4dfc62fb","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318203241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48392024-02-13T21:41:16.516Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318215866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314,"session_id":"06c6ef0b-a452-4e87-9215-8d2e4dfc62fb"}
48402024-02-13T21:41:16.516Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:15.318229291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
48412024-02-13T21:41:16.516ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48422024-02-13T21:41:16.516ZLoading write count information from file "/tmp/test_repair_verify.data"
48432024-02-13T21:41:16.516ZRun Repair workload
48442024-02-13T21:41:16.516Z01/30 Write block 458 len 20480 data: 4 5 5 4 4
48452024-02-13T21:41:16.516Z02/30 Write block 494 len 28672 data: 7 7 7 7 8 6 5
48462024-02-13T21:41:16.516Z03/30 Write block 194 len 40960 data: 6 5 5 5 6 6 5 5 5 5
48472024-02-13T21:41:16.516Z04/30 Write block 519 len 40960 data: 10 11 10 9 7 4 4 3 3 3
48482024-02-13T21:41:16.516Z05/30 Read block 175 len 24576
48492024-02-13T21:41:16.520Z06/30 Write block 568 len 8192 data: 7 5
48502024-02-13T21:41:16.520Z07/30 Read block 189 len 16384
48512024-02-13T21:41:16.520Z08/30 Read block 299 len 32768
48522024-02-13T21:41:16.520Z09/30 Flush
48532024-02-13T21:41:16.523Z10/30 Write block 471 len 16384 data: 7 7 7 7
48542024-02-13T21:41:16.523Z11/30 Read block 480 len 32768
48552024-02-13T21:41:16.523Z12/30 Write block 49 len 4096 data: 6
48562024-02-13T21:41:16.523Z13/30 Write block 61 len 4096 data: 6
48572024-02-13T21:41:16.523Z14/30 Read block 239 len 40960
48582024-02-13T21:41:16.527Z15/30 Flush
48592024-02-13T21:41:16.527Z16/30 Write block 516 len 20480 data: 5 7 9 11 12
48602024-02-13T21:41:16.527Z17/30 Read block 14 len 28672
48612024-02-13T21:41:16.530Z18/30 Flush
48622024-02-13T21:41:16.533Z19/30 Write block 584 len 12288 data: 5 6 5
48632024-02-13T21:41:16.536Z20/30 Write block 97 len 16384 data: 4 4 4 4
48642024-02-13T21:41:16.536Z21/30 Read block 476 len 4096
48652024-02-13T21:41:16.536Z22/30 Read block 571 len 8192
48662024-02-13T21:41:16.536Z23/30 Read block 127 len 8192
48672024-02-13T21:41:16.536Z24/30 Flush
48682024-02-13T21:41:16.539Z25/30 Write block 400 len 24576 data: 2 2 2 2 2 2
48692024-02-13T21:41:16.540Z26/30 Read block 551 len 40960
48702024-02-13T21:41:16.543Z27/30 Read block 64 len 20480
48712024-02-13T21:41:16.543Z28/30 Write block 194 len 4096 data: 7
48722024-02-13T21:41:16.544Z29/30 Write block 239 len 20480 data: 7 7 7 6 5
48732024-02-13T21:41:16.544Z30/30 Write block 381 len 4096 data: 7
48742024-02-13T21:41:16.544Z----------------------------------------------------------------
48752024-02-13T21:41:16.544Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48762024-02-13T21:41:16.544ZStates: Active Active Active
48772024-02-13T21:41:16.544ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48782024-02-13T21:41:16.544Z 1 Acked 1000 Write 5 Done Done Sent false
48792024-02-13T21:41:16.544Z 2 Acked 1001 Write 7 Done Done Sent false
48802024-02-13T21:41:16.544Z 3 Acked 1002 Write 10 Done Done Sent false
48812024-02-13T21:41:16.544Z 4 Acked 1003 Write 10 Done Done Sent false
48822024-02-13T21:41:16.544Z 5 Acked 1004 Read 6 Done Done Sent false
48832024-02-13T21:41:16.544Z 6 Acked 1005 Write 2 Done Done Sent false
48842024-02-13T21:41:16.544Z 7 Acked 1006 Read 4 Done Done Sent false
48852024-02-13T21:41:16.544Z 8 Acked 1007 Read 8 Done Done Sent false
48862024-02-13T21:41:16.544Z 9 Acked 1008 Flush 0 Done Done Sent false
48872024-02-13T21:41:16.544Z 10 Acked 1009 Write 4 Done Done Sent false
48882024-02-13T21:41:16.544Z 11 Acked 1010 Read 8 Done Done Sent false
48892024-02-13T21:41:16.544Z 12 Acked 1011 Write 1 Done Done Sent false
48902024-02-13T21:41:16.544Z 13 Acked 1012 Write 1 Done Done Sent false
48912024-02-13T21:41:16.544Z 14 Acked 1013 Read 10 Done Done Sent false
48922024-02-13T21:41:16.544Z 15 Acked 1014 Flush 0 Done Done Sent false
48932024-02-13T21:41:16.544Z 16 Acked 1015 Write 5 Done Done Sent false
48942024-02-13T21:41:16.544Z 17 Acked 1016 Read 7 Done Done Sent false
48952024-02-13T21:41:16.545Z 18 Acked 1017 Flush 0 Done Done Sent false
48962024-02-13T21:41:16.545Z 19 Acked 1018 Write 3 Done Done Sent false
48972024-02-13T21:41:16.545Z 20 Acked 1019 Write 4 Done Done Sent false
48982024-02-13T21:41:16.545Z 21 Acked 1020 Read 1 Done Done Sent false
48992024-02-13T21:41:16.545Z 22 Acked 1021 Read 2 Done Done Sent false
49002024-02-13T21:41:16.545Z 23 Acked 1022 Read 2 Done Done Sent false
49012024-02-13T21:41:16.545Z 24 Acked 1023 Flush 0 Done Done Sent false
49022024-02-13T21:41:16.545Z 25 Acked 1024 Write 6 Done Done Sent false
49032024-02-13T21:41:16.545Z 26 Acked 1025 Read 10 Done Done Sent false
49042024-02-13T21:41:16.545Z 27 Acked 1026 Read 5 Done Done Sent false
49052024-02-13T21:41:16.545Z 28 Acked 1027 Write 1 Done Done Sent false
49062024-02-13T21:41:16.545Z 29 Acked 1028 Write 5 Sent Sent Sent false
49072024-02-13T21:41:16.545Z 30 Acked 1029 Write 1 Sent Sent Sent false
49082024-02-13T21:41:16.545Z STATES DS:0 DS:1 DS:2 TOTAL
49092024-02-13T21:41:16.545Z New 0 0 0 0
49102024-02-13T21:41:16.545Z Sent 2 2 30 34
49112024-02-13T21:41:16.545Z Done 28 28 0 56
49122024-02-13T21:41:16.545Z Skipped 0 0 0 0
49132024-02-13T21:41:16.545Z Error 0 0 0 0
49142024-02-13T21:41:16.545ZLast Flush: 1023 1023 0
49152024-02-13T21:41:16.545ZDownstairs last five completed:
49162024-02-13T21:41:16.546ZUpstairs last five completed: 30 29 28 27 26
49172024-02-13T21:41:16.549Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:15.353186761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1314}
49182024-02-13T21:41:16.549ZWrote out file "/tmp/test_repair_verify.data"
49192024-02-13T21:41:16.549Z
49202024-02-13T21:41:16.553ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
49212024-02-13T21:41:16.567ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49222024-02-13T21:41:16.568Z 0 000-019 42 42 42 65 65 65 F F F
49232024-02-13T21:41:16.568Z 1 020-039 36 36 36 60 60 60 F F F
49242024-02-13T21:41:16.568Z 2 040-059 44 44 39 68 68 62 F F F <---
49252024-02-13T21:41:16.568Z 3 060-079 44 44 40 68 68 63 F F F <---
49262024-02-13T21:41:16.568Z 4 080-099 44 44 40 70 70 63 F F F <---
49272024-02-13T21:41:16.568Z 5 100-119 44 44 43 70 70 66 F F F <---
49282024-02-13T21:41:16.569Z 6 120-139 42 42 42 65 65 65 F F F
49292024-02-13T21:41:16.569Z 7 140-159 42 42 42 65 65 65 F F F
49302024-02-13T21:41:16.569Z 8 160-179 39 39 39 62 62 62 F F F
49312024-02-13T21:41:16.569Z 9 180-199 44 44 34 67 67 55 T T F <---
49322024-02-13T21:41:16.569Z 10 200-219 44 44 42 67 67 65 F F F <---
49332024-02-13T21:41:16.573Z 11 220-239 41 41 41 64 64 64 T T F <---
49342024-02-13T21:41:16.573Z 12 240-259 42 42 42 65 65 65 T T F <---
49352024-02-13T21:41:16.574Z 13 260-279 40 40 40 63 63 63 F F F
49362024-02-13T21:41:16.574Z 14 280-299 42 42 42 65 65 65 F F F
49372024-02-13T21:41:16.574Z 15 300-319 41 41 41 64 64 64 F F F
49382024-02-13T21:41:16.574Z 16 320-339 42 42 42 65 65 65 F F F
49392024-02-13T21:41:16.574Z 17 340-359 42 42 42 65 65 65 F F F
49402024-02-13T21:41:16.574Z 18 360-379 39 39 39 62 62 62 F F F
49412024-02-13T21:41:16.574Z 19 380-399 38 38 38 61 61 61 T T F <---
49422024-02-13T21:41:16.574Z 20 400-419 41 41 41 64 64 64 T T F <---
49432024-02-13T21:41:16.574Z 21 420-439 41 41 41 64 64 64 F F F
49442024-02-13T21:41:16.575Z 22 440-459 44 44 39 67 67 62 F F F <---
49452024-02-13T21:41:16.575Z 23 460-479 44 44 43 68 68 66 F F F <---
49462024-02-13T21:41:16.575Z 24 480-499 44 44 42 67 67 65 F F F <---
49472024-02-13T21:41:16.575Z 25 500-519 44 44 43 69 69 66 F F F <---
49482024-02-13T21:41:16.575Z 26 520-539 44 44 43 69 69 66 F F F <---
49492024-02-13T21:41:16.575Z 27 540-559 43 43 43 66 66 66 F F F
49502024-02-13T21:41:16.575Z 28 560-579 44 44 40 67 67 63 F F F <---
49512024-02-13T21:41:16.575Z 29 580-599 44 44 39 70 70 62 F F F <---
49522024-02-13T21:41:16.575ZMax gen: 44, Max flush: 70
49532024-02-13T21:41:16.576ZError: Difference in extent metadata found!
49542024-02-13T21:41:16.576ZA Difference in extent metadata is expected here
49552024-02-13T21:41:16.576ZOn loop 22
49562024-02-13T21:41:16.576Z
49572024-02-13T21:41:16.576ZVerifying data now
49582024-02-13T21:41:21.364ZNo change after verify
49592024-02-13T21:41:21.364ZLoop: 22 Downstairs dump after verify (and repair):
49602024-02-13T21:41:21.378ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49612024-02-13T21:41:21.378Z 0 000-019 42 42 42 65 65 65 F F F
49622024-02-13T21:41:21.378Z 1 020-039 36 36 36 60 60 60 F F F
49632024-02-13T21:41:21.379Z 2 040-059 44 44 44 68 68 68 F F F
49642024-02-13T21:41:21.379Z 3 060-079 44 44 44 68 68 68 F F F
49652024-02-13T21:41:21.379Z 4 080-099 44 44 44 70 70 70 F F F
49662024-02-13T21:41:21.379Z 5 100-119 44 44 44 70 70 70 F F F
49672024-02-13T21:41:21.379Z 6 120-139 42 42 42 65 65 65 F F F
49682024-02-13T21:41:21.379Z 7 140-159 42 42 42 65 65 65 F F F
49692024-02-13T21:41:21.383Z 8 160-179 39 39 39 62 62 62 F F F
49702024-02-13T21:41:21.383Z 9 180-199 45 45 45 71 71 71 F F F
49712024-02-13T21:41:21.383Z 10 200-219 44 44 44 67 67 67 F F F
49722024-02-13T21:41:21.383Z 11 220-239 45 45 45 71 71 71 F F F
49732024-02-13T21:41:21.384Z 12 240-259 45 45 45 71 71 71 F F F
49742024-02-13T21:41:21.384Z 13 260-279 40 40 40 63 63 63 F F F
49752024-02-13T21:41:21.384Z 14 280-299 42 42 42 65 65 65 F F F
49762024-02-13T21:41:21.384Z 15 300-319 41 41 41 64 64 64 F F F
49772024-02-13T21:41:21.384Z 16 320-339 42 42 42 65 65 65 F F F
49782024-02-13T21:41:21.384Z 17 340-359 42 42 42 65 65 65 F F F
49792024-02-13T21:41:21.384Z 18 360-379 39 39 39 62 62 62 F F F
49802024-02-13T21:41:21.384Z 19 380-399 45 45 45 71 71 71 F F F
49812024-02-13T21:41:21.384Z 20 400-419 45 45 45 71 71 71 F F F
49822024-02-13T21:41:21.384Z 21 420-439 41 41 41 64 64 64 F F F
49832024-02-13T21:41:21.384Z 22 440-459 44 44 44 67 67 67 F F F
49842024-02-13T21:41:21.384Z 23 460-479 44 44 44 68 68 68 F F F
49852024-02-13T21:41:21.385Z 24 480-499 44 44 44 67 67 67 F F F
49862024-02-13T21:41:21.385Z 25 500-519 44 44 44 69 69 69 F F F
49872024-02-13T21:41:21.385Z 26 520-539 44 44 44 69 69 69 F F F
49882024-02-13T21:41:21.385Z 27 540-559 43 43 43 66 66 66 F F F
49892024-02-13T21:41:21.385Z 28 560-579 44 44 44 67 67 67 F F F
49902024-02-13T21:41:21.385Z 29 580-599 44 44 44 70 70 70 F F F
49912024-02-13T21:41:21.385ZMax gen: 45, Max flush: 71
49922024-02-13T21:41:21.385Z
49932024-02-13T21:41:21.385ZBegin loop 23
49942024-02-13T21:41:21.385ZDownstairs to restart: 1
49952024-02-13T21:41:21.397Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.20423458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
49962024-02-13T21:41:21.397Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\nCrucible runtime is spawned
49972024-02-13T21:41:21.404Z git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204588262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
49982024-02-13T21:41:21.404Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204633216Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
49992024-02-13T21:41:21.404Z{"msg":"Crucible a6994440-64d4-414a-b31d-8370d4ad12ed has session id: acce9ef7-9af2-4b26-b74f-33a1f86be845","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204662205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50002024-02-13T21:41:21.404Z{"msg":"Upstairs opts: Upstairs UUID: a6994440-64d4-414a-b31d-8370d4ad12ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204711318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50012024-02-13T21:41:21.404Z{"msg":"Crucible stats registered with UUID: a6994440-64d4-414a-b31d-8370d4ad12ed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204776314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50022024-02-13T21:41:21.404Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204828395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
50032024-02-13T21:41:21.404Z{"msg":"a6994440-64d4-414a-b31d-8370d4ad12ed active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204858184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50042024-02-13T21:41:21.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204900339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50052024-02-13T21:41:21.404Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204948991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50062024-02-13T21:41:21.404Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.204996074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50072024-02-13T21:41:21.405Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205030862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50082024-02-13T21:41:21.405Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205058741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50092024-02-13T21:41:21.405Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205085502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50102024-02-13T21:41:21.405Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205113801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50112024-02-13T21:41:21.405Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205180128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50122024-02-13T21:41:21.405Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.20522666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50132024-02-13T21:41:21.405Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.2052553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50142024-02-13T21:41:21.405Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205279381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50152024-02-13T21:41:21.405Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205303743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"io task","client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50162024-02-13T21:41:21.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205791056Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50172024-02-13T21:41:21.405Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205826733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50182024-02-13T21:41:21.405Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.205852334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50192024-02-13T21:41:21.405Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.20588978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50202024-02-13T21:41:21.406Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.206399215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50212024-02-13T21:41:21.406Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.20655099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50222024-02-13T21:41:21.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207028037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50232024-02-13T21:41:21.406Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207066553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50242024-02-13T21:41:21.406Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207092744Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50252024-02-13T21:41:21.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207124992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50262024-02-13T21:41:21.406Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207149683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50272024-02-13T21:41:21.406Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207177223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50282024-02-13T21:41:21.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207199505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50292024-02-13T21:41:21.406Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207226775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50302024-02-13T21:41:21.406Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.207579567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50312024-02-13T21:41:21.406Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208009801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50322024-02-13T21:41:21.406Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208049807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50332024-02-13T21:41:21.407Z{"msg":"[0]R flush_numbers[0..12]: [65, 60, 68, 68, 70, 70, 65, 65, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208066341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50342024-02-13T21:41:21.407Z{"msg":"[0]R generation[0..12]: [42, 36, 44, 44, 44, 44, 42, 42, 39, 45, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208088133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50352024-02-13T21:41:21.407Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208103837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50362024-02-13T21:41:21.407Z{"msg":"[1]R flush_numbers[0..12]: [65, 60, 68, 68, 70, 70, 65, 65, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208124789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50372024-02-13T21:41:21.407Z{"msg":"[1]R generation[0..12]: [42, 36, 44, 44, 44, 44, 42, 42, 39, 45, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208139044Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50382024-02-13T21:41:21.407Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208160167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50392024-02-13T21:41:21.407Z{"msg":"[2]R flush_numbers[0..12]: [65, 60, 68, 68, 70, 70, 65, 65, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208174521Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50402024-02-13T21:41:21.407Z{"msg":"[2]R generation[0..12]: [42, 36, 44, 44, 44, 44, 42, 42, 39, 45, 44, 45]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
50412024-02-13T21:41:21.407Z,"time":"2024-02-13T21:41:20.208188756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50422024-02-13T21:41:21.407Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.2082051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50432024-02-13T21:41:21.407Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208218685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50442024-02-13T21:41:21.407Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208239038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50452024-02-13T21:41:21.407Z{"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208252363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50462024-02-13T21:41:21.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208272796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"":"downstairs","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50472024-02-13T21:41:21.408Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208286801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50482024-02-13T21:41:21.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208298846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"0","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50492024-02-13T21:41:21.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208318699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"1","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50502024-02-13T21:41:21.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208332244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"client":"2","":"downstairs","session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50512024-02-13T21:41:21.408Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208345309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50522024-02-13T21:41:21.408Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208357835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50532024-02-13T21:41:21.412Z{"msg":"a6994440-64d4-414a-b31d-8370d4ad12ed is now active with session: acce9ef7-9af2-4b26-b74f-33a1f86be845","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208378407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50542024-02-13T21:41:21.412Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208411845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322,"session_id":"acce9ef7-9af2-4b26-b74f-33a1f86be845"}
50552024-02-13T21:41:21.412Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:20.208446573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
50562024-02-13T21:41:21.412ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50572024-02-13T21:41:21.412ZLoading write count information from file "/tmp/test_repair_verify.data"
50582024-02-13T21:41:21.412ZRun Repair workload
50592024-02-13T21:41:21.412Z01/30 Flush
50602024-02-13T21:41:21.412Z02/30 Write block 500 len 12288 data: 6 3 2
50612024-02-13T21:41:21.413Z03/30 Read block 434 len 4096
50622024-02-13T21:41:21.413Z04/30 Read block 275 len 12288
50632024-02-13T21:41:21.413Z05/30 Write block 312 len 40960 data: 6 4 4 4 4 4 3 3 3 3
50642024-02-13T21:41:21.413Z06/30 Write block 518 len 24576 data: 10 12 13 11 10 8
50652024-02-13T21:41:21.413Z07/30 Write block 332 len 32768 data: 6 6 6 7 7 6 6 5
50662024-02-13T21:41:21.413Z08/30 Read block 269 len 4096
50672024-02-13T21:41:21.413Z09/30 Read block 310 len 40960
50682024-02-13T21:41:21.413Z10/30 Read block 276 len 20480
50692024-02-13T21:41:21.413Z11/30 Flush
50702024-02-13T21:41:21.416Z12/30 Write block 294 len 12288 data: 5 5 5
50712024-02-13T21:41:21.416Z13/30 Write block 486 len 12288 data: 4 4 5
50722024-02-13T21:41:21.416Z14/30 Read block 406 len 40960
50732024-02-13T21:41:21.416Z15/30 Read block 436 len 40960
50742024-02-13T21:41:21.416Z16/30 Write block 57 len 24576 data: 6 6 6 6 7 6
50752024-02-13T21:41:21.416Z17/30 Read block 358 len 28672
50762024-02-13T21:41:21.419Z18/30 Read block 173 len 20480
50772024-02-13T21:41:21.419Z19/30 Flush
50782024-02-13T21:41:21.422Z20/30 Write block 581 len 12288 data: 3 4 6
50792024-02-13T21:41:21.422Z21/30 Write block 352 len 12288 data: 3 4 5
50802024-02-13T21:41:21.423Z22/30 Write block 276 len 32768 data: 4 5 5 5 4 2 2 2
50812024-02-13T21:41:21.430Z23/30 Write block 275 len 4096 data: 4
50822024-02-13T21:41:21.430Z24/30 Write block 456 len 16384 data: 4 4 5 6
50832024-02-13T21:41:21.430Z25/30 Write block 515 len 40960 data: 7 6 8 11 13 14 12 11 9 5
50842024-02-13T21:41:21.430Z26/30 Write block 153 len 28672 data: 3 4 5 5 6 6 3
50852024-02-13T21:41:21.430Z27/30 Write block 351 len 8192 data: 3 4
50862024-02-13T21:41:21.430Z28/30 Write block 77 len 20480 data: 4 4 5 5 5
50872024-02-13T21:41:21.431Z29/30 Write block 355 len 32768 data: 6 7 7 6 6 5 3 3
50882024-02-13T21:41:21.431Z30/30 Write block 409 len 24576 data: 2 2 3 3 4 5
50892024-02-13T21:41:21.431Z----------------------------------------------------------------
50902024-02-13T21:41:21.431Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:29
50912024-02-13T21:41:21.431ZStates: Active Active Active
50922024-02-13T21:41:21.431ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50932024-02-13T21:41:21.431Z 2 Acked 1001 Write 3 Done Done Done false
50942024-02-13T21:41:21.431Z 3 Acked 1002 Read 1 Done Done Done false
50952024-02-13T21:41:21.431Z 4 Acked 1003 Read 3 Done Sent Done false
50962024-02-13T21:41:21.431Z 5 Acked 1004 Write 10 Done Sent Done false
50972024-02-13T21:41:21.431Z 6 Acked 1005 Write 6 Done Sent Done false
50982024-02-13T21:41:21.431Z 7 Acked 1006 Write 8 Done Sent Done false
50992024-02-13T21:41:21.431Z 8 Acked 1007 Read 1 Done Sent Done false
51002024-02-13T21:41:21.431Z 9 Acked 1008 Read 10 Done Sent Done false
51012024-02-13T21:41:21.431Z 10 Acked 1009 Read 5 Done Sent Done false
51022024-02-13T21:41:21.431Z 11 Acked 1010 Flush 0 Done Sent Done false
51032024-02-13T21:41:21.431Z 12 Acked 1011 Write 3 Done Sent Done false
51042024-02-13T21:41:21.431Z 13 Acked 1012 Write 3 Done Sent Done false
51052024-02-13T21:41:21.431Z 14 Acked 1013 Read 10 Done Sent Done false
51062024-02-13T21:41:21.431Z 15 Acked 1014 Read 10 Done Sent Done false
51072024-02-13T21:41:21.431Z 16 Acked 1015 Write 6 Done Sent Done false
51082024-02-13T21:41:21.431Z 17 Acked 1016 Read 7 Done Sent Done false
51092024-02-13T21:41:21.431Z 18 Acked 1017 Read 5 Done Sent Done false
51102024-02-13T21:41:21.431Z 19 Acked 1018 Flush 0 Done Sent Done false
51112024-02-13T21:41:21.431Z 20 Acked 1019 Write 3 Done Sent Done false
51122024-02-13T21:41:21.432Z 21 Acked 1020 Write 3 Done Sent Done false
51132024-02-13T21:41:21.432Z 22 Acked 1021 Write 8 Done Sent Done false
51142024-02-13T21:41:21.432Z 23 Acked 1022 Write 1 Done Sent Done false
51152024-02-13T21:41:21.432Z 24 Acked 1023 Write 4 Done Sent Done false
51162024-02-13T21:41:21.432Z 25 Acked 1024 Write 10 Done Sent Done false
51172024-02-13T21:41:21.432Z 26 Acked 1025 Write 7 Done Sent Done false
51182024-02-13T21:41:21.432Z 27 Acked 1026 Write 2 Done Sent Done false
51192024-02-13T21:41:21.432Z 28 Acked 1027 Write 5 Done Sent Done false
51202024-02-13T21:41:21.432Z 29 Acked 1028 Write 8 Done Sent Sent false
51212024-02-13T21:41:21.432Z 30 Acked 1029 Write 6 Sent Sent Sent false
51222024-02-13T21:41:21.432Z STATES DS:0 DS:1 DS:2 TOTAL
51232024-02-13T21:41:21.432Z New 0 0 0 0
51242024-02-13T21:41:21.432Z Sent 1 27 2 30
51252024-02-13T21:41:21.432Z Done 28 2 27 57
51262024-02-13T21:41:21.432Z Skipped 0 0 0 0
51272024-02-13T21:41:21.432Z Error 0 0 0 0
51282024-02-13T21:41:21.432ZLast Flush: 1018 1000 1018
51292024-02-13T21:41:21.432ZDownstairs last five completed: 1000
51302024-02-13T21:41:21.432ZUpstairs last five completed: 30 29 28 27 26
51312024-02-13T21:41:21.432Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:20.236927901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1322}
51322024-02-13T21:41:21.432ZWrote out file "/tmp/test_repair_verify.data"
51332024-02-13T21:41:21.435Z
51342024-02-13T21:41:21.439ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51352024-02-13T21:41:21.455ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51362024-02-13T21:41:21.455Z 0 000-019 42 42 42 65 65 65 F F F
51372024-02-13T21:41:21.455Z 1 020-039 36 36 36 60 60 60 F F F
51382024-02-13T21:41:21.455Z 2 040-059 46 44 46 74 68 74 F F F <---
51392024-02-13T21:41:21.455Z 3 060-079 46 44 46 74 68 74 T F T <---
51402024-02-13T21:41:21.455Z 4 080-099 44 44 44 70 70 70 T F T <---
51412024-02-13T21:41:21.455Z 5 100-119 44 44 44 70 70 70 F F F
51422024-02-13T21:41:21.455Z 6 120-139 42 42 42 65 65 65 F F F
51432024-02-13T21:41:21.455Z 7 140-159 42 42 42 65 65 65 T F T <---
51442024-02-13T21:41:21.455Z 8 160-179 39 39 39 62 62 62 F F F
51452024-02-13T21:41:21.455Z 9 180-199 45 45 45 71 71 71 F F F
51462024-02-13T21:41:21.455Z 10 200-219 44 44 44 67 67 67 F F F
51472024-02-13T21:41:21.459Z 11 220-239 45 45 45 71 71 71 F F F
51482024-02-13T21:41:21.459Z 12 240-259 45 45 45 71 71 71 F F F
51492024-02-13T21:41:21.459Z 13 260-279 40 40 40 63 63 63 T F T <---
51502024-02-13T21:41:21.459Z 14 280-299 46 42 46 74 65 74 T F T <---
51512024-02-13T21:41:21.459Z 15 300-319 46 41 46 73 64 73 F F F <---
51522024-02-13T21:41:21.459Z 16 320-339 46 42 46 73 65 73 F F F <---
51532024-02-13T21:41:21.459Z 17 340-359 42 42 42 65 65 65 T F T <---
51542024-02-13T21:41:21.459Z 18 360-379 39 39 39 62 62 62 T F T <---
51552024-02-13T21:41:21.459Z 19 380-399 45 45 45 71 71 71 F F F
51562024-02-13T21:41:21.459Z 20 400-419 45 45 45 71 71 71 T F T <---
51572024-02-13T21:41:21.459Z 21 420-439 41 41 41 64 64 64 F F F
51582024-02-13T21:41:21.459Z 22 440-459 44 44 44 67 67 67 T F T <---
51592024-02-13T21:41:21.459Z 23 460-479 44 44 44 68 68 68 F F F
51602024-02-13T21:41:21.459Z 24 480-499 46 44 46 74 67 74 F F F <---
51612024-02-13T21:41:21.459Z 25 500-519 46 44 46 73 69 73 T T T <---
51622024-02-13T21:41:21.459Z 26 520-539 46 44 46 73 69 73 T F T <---
51632024-02-13T21:41:21.459Z 27 540-559 43 43 43 66 66 66 F F F
51642024-02-13T21:41:21.459Z 28 560-579 44 44 44 67 67 67 F F F
51652024-02-13T21:41:21.459Z 29 580-599 44 44 44 70 70 70 T F T <---
51662024-02-13T21:41:21.460ZMax gen: 46, Max flush: 74
51672024-02-13T21:41:21.460ZError: Difference in extent metadata found!
51682024-02-13T21:41:21.460ZA Difference in extent metadata is expected here
51692024-02-13T21:41:21.460ZOn loop 23
51702024-02-13T21:41:21.460Z
51712024-02-13T21:41:21.463ZVerifying data now
51722024-02-13T21:41:26.252ZNo change after verify
51732024-02-13T21:41:26.256ZLoop: 23 Downstairs dump after verify (and repair):
51742024-02-13T21:41:26.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51752024-02-13T21:41:26.267Z 0 000-019 42 42 42 65 65 65 F F F
51762024-02-13T21:41:26.267Z 1 020-039 36 36 36 60 60 60 F F F
51772024-02-13T21:41:26.267Z 2 040-059 46 46 46 74 74 74 F F F
51782024-02-13T21:41:26.267Z 3 060-079 47 47 47 75 75 75 F F F
51792024-02-13T21:41:26.268Z 4 080-099 47 47 47 75 75 75 F F F
51802024-02-13T21:41:26.268Z 5 100-119 44 44 44 70 70 70 F F F
51812024-02-13T21:41:26.268Z 6 120-139 42 42 42 65 65 65 F F F
51822024-02-13T21:41:26.268Z 7 140-159 47 47 47 75 75 75 F F F
51832024-02-13T21:41:26.268Z 8 160-179 39 39 39 62 62 62 F F F
51842024-02-13T21:41:26.268Z 9 180-199 45 45 45 71 71 71 F F F
51852024-02-13T21:41:26.268Z 10 200-219 44 44 44 67 67 67 F F F
51862024-02-13T21:41:26.268Z 11 220-239 45 45 45 71 71 71 F F F
51872024-02-13T21:41:26.273Z 12 240-259 45 45 45 71 71 71 F F F
51882024-02-13T21:41:26.273Z 13 260-279 47 47 47 75 75 75 F F F
51892024-02-13T21:41:26.274Z 14 280-299 47 47 47 75 75 75 F F F
51902024-02-13T21:41:26.274Z 15 300-319 46 46 46 73 73 73 F F F
51912024-02-13T21:41:26.274Z 16 320-339 46 46 46 73 73 73 F F F
51922024-02-13T21:41:26.274Z 17 340-359 47 47 47 75 75 75 F F F
51932024-02-13T21:41:26.274Z 18 360-379 47 47 47 75 75 75 F F F
51942024-02-13T21:41:26.274Z 19 380-399 45 45 45 71 71 71 F F F
51952024-02-13T21:41:26.274Z 20 400-419 47 47 47 75 75 75 F F F
51962024-02-13T21:41:26.275Z 21 420-439 41 41 41 64 64 64 F F F
51972024-02-13T21:41:26.275Z 22 440-459 47 47 47 75 75 75 F F F
51982024-02-13T21:41:26.275Z 23 460-479 44 44 44 68 68 68 F F F
51992024-02-13T21:41:26.275Z 24 480-499 46 46 46 74 74 74 F F F
52002024-02-13T21:41:26.275Z 25 500-519 47 47 47 75 75 75 F F F
52012024-02-13T21:41:26.275Z 26 520-539 47 47 47 75 75 75 F F F
52022024-02-13T21:41:26.275Z 27 540-559 43 43 43 66 66 66 F F F
52032024-02-13T21:41:26.275Z 28 560-579 44 44 44 67 67 67 F F F
52042024-02-13T21:41:26.275Z 29 580-599 47 47 47 75 75 75 F F F
52052024-02-13T21:41:26.276ZMax gen: 47, Max flush: 75
52062024-02-13T21:41:26.276Z
52072024-02-13T21:41:26.276ZBegin loop 24
52082024-02-13T21:41:26.276ZDownstairs to restart: 1
52092024-02-13T21:41:26.302Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094521737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
52102024-02-13T21:41:26.302Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094749344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
52112024-02-13T21:41:26.302Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094769107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
52122024-02-13T21:41:26.310Z{"msg":"Crucible c4e266f5-d123-403b-811c-c1d60d0ccaf9 has session id: 3893a96d-d807-4ee7-8bdc-8846424c24a1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094802665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52132024-02-13T21:41:26.310ZCrucible runtime is spawned
52142024-02-13T21:41:26.310Z{"msg":"Upstairs opts: Upstairs UUID: c4e266f5-d123-403b-811c-c1d60d0ccaf9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094856506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52152024-02-13T21:41:26.310Z{"msg":"Crucible stats registered with UUID: c4e266f5-d123-403b-811c-c1d60d0ccaf9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094895541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52162024-02-13T21:41:26.310Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.094918223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
52172024-02-13T21:41:26.311Z{"msg":"c4e266f5-d123-403b-811c-c1d60d0ccaf9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095022385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52182024-02-13T21:41:26.311Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095051245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52192024-02-13T21:41:26.311Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095066959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52202024-02-13T21:41:26.311Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095096168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52212024-02-13T21:41:26.311Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095113512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52222024-02-13T21:41:26.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095132475Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52232024-02-13T21:41:26.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095156926Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52242024-02-13T21:41:26.312Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095190294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52252024-02-13T21:41:26.312Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095227391Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52262024-02-13T21:41:26.312Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095262988Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52272024-02-13T21:41:26.312Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095375047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52282024-02-13T21:41:26.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095436525Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52292024-02-13T21:41:26.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.095466284Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"io task","client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52302024-02-13T21:41:26.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.096058989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52312024-02-13T21:41:26.313Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.096094666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52322024-02-13T21:41:26.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.09611096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52332024-02-13T21:41:26.313Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.096191531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52342024-02-13T21:41:26.313Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.096634241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52352024-02-13T21:41:26.313Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.096793233Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52362024-02-13T21:41:26.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097130421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52372024-02-13T21:41:26.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097170076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52382024-02-13T21:41:26.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.09718754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52392024-02-13T21:41:26.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.0972155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52402024-02-13T21:41:26.314Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097236252Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52412024-02-13T21:41:26.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097263772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52422024-02-13T21:41:26.315Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097394365Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52432024-02-13T21:41:26.315Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097429652Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52442024-02-13T21:41:26.315Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.097908818Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52452024-02-13T21:41:26.315Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098329586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52462024-02-13T21:41:26.315Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098367952Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52472024-02-13T21:41:26.315Z{"msg":"[0]R flush_numbers[0..12]: [65, 60, 74, 75, 75, 70, 65, 75, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098384376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52482024-02-13T21:41:26.315Z{"msg":"[0]R generation[0..12]: [42, 36, 46, 47, 47, 44, 42, 47, 39, 45, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098416584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52492024-02-13T21:41:26.315Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098443534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52502024-02-13T21:41:26.315Z{"msg":"[1]R flush_numbers[0..12]: [65, 60, 74, 75, 75, 70, 65, 75, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098471804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52512024-02-13T21:41:26.316ZWait for a query_work_queue command to finish before sending IO
52522024-02-13T21:41:26.316Z{"msg":"[1]R generation[0..12]: [42, 36, 46, 47, 47, 44, 42, 47, 39, 45, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098499594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52532024-02-13T21:41:26.316Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098529183Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52542024-02-13T21:41:26.316Z{"msg":"[2]R flush_numbers[0..12]: [65, 60, 74, 75, 75, 70, 65, 75, 62, 71, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098553674Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52552024-02-13T21:41:26.316Z{"msg":"[2]R generation[0..12]: [42, 36, 46, 47, 47, 44, 42, 47, 39, 45, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098575086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52562024-02-13T21:41:26.316Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098606155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52572024-02-13T21:41:26.316Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098625558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52582024-02-13T21:41:26.317Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098639763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52592024-02-13T21:41:26.317Z{"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098653758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52602024-02-13T21:41:26.317Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098667013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"":"downstairs","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52612024-02-13T21:41:26.317Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098681168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52622024-02-13T21:41:26.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098694034Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"0","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52632024-02-13T21:41:26.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098708168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"1","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52642024-02-13T21:41:26.317Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098721783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"client":"2","":"downstairs","session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52652024-02-13T21:41:26.318Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098743485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52662024-02-13T21:41:26.318Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098756211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52672024-02-13T21:41:26.318Z{"msg":"c4e266f5-d123-403b-811c-c1d60d0ccaf9 is now active with session: 3893a96d-d807-4ee7-8bdc-8846424c24a1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098769166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52682024-02-13T21:41:26.318Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098787809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330,"session_id":"3893a96d-d807-4ee7-8bdc-8846424c24a1"}
52692024-02-13T21:41:26.318Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:25.098800295Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
52702024-02-13T21:41:26.318ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52712024-02-13T21:41:26.318ZLoading write count information from file "/tmp/test_repair_verify.data"
52722024-02-13T21:41:26.319ZRun Repair workload
52732024-02-13T21:41:26.319Z01/30 Write block 347 len 28672 data: 5 4 2 2 4 5 5
52742024-02-13T21:41:26.319Z02/30 Write block 538 len 40960 data: 5 5 6 7 5 5 6 8 7 6
52752024-02-13T21:41:26.319Z03/30 Write block 32 len 40960 data: 6 5 6 7 7 5 5 5 5 4
52762024-02-13T21:41:26.319Z04/30 Write block 162 len 12288 data: 4 4 3
52772024-02-13T21:41:26.319Z05/30 Read block 464 len 28672
52782024-02-13T21:41:26.320Z06/30 Read block 416 len 24576
52792024-02-13T21:41:26.320Z07/30 Write block 158 len 40960 data: 7 4 3 4 5 5 4 2 3 2
52802024-02-13T21:41:26.320Z08/30 Write block 32 len 28672 data: 7 6 7 8 8 6 6
52812024-02-13T21:41:26.320Z09/30 Write block 22 len 4096 data: 6
52822024-02-13T21:41:26.320Z10/30 Read block 220 len 8192
52832024-02-13T21:41:26.320Z11/30 Read block 36 len 16384
52842024-02-13T21:41:26.320Z12/30 Write block 540 len 20480 data: 7 8 6 6 7
52852024-02-13T21:41:26.320Z13/30 Write block 111 len 8192 data: 3 3
52862024-02-13T21:41:26.321Z14/30 Read block 282 len 20480
52872024-02-13T21:41:26.326Z15/30 Write block 195 len 20480 data: 6 6 6 7 7
52882024-02-13T21:41:26.327Z16/30 Read block 519 len 8192
52892024-02-13T21:41:26.327Z17/30 Read block 593 len 12288
52902024-02-13T21:41:26.327Z18/30 Write block 177 len 16384 data: 4 3 3 3
52912024-02-13T21:41:26.327Z19/30 Write block 81 len 24576 data: 6 5 6 5 5 4
52922024-02-13T21:41:26.327Z20/30 Read block 337 len 40960
52932024-02-13T21:41:26.327Z21/30 Read block 45 len 8192
52942024-02-13T21:41:26.327Z22/30 Write block 190 len 8192 data: 7 7
52952024-02-13T21:41:26.328Z23/30 Read block 211 len 24576
52962024-02-13T21:41:26.328Z24/30 Write block 72 len 16384 data: 5 3 4 4
52972024-02-13T21:41:26.328Z25/30 Write block 327 len 4096 data: 4
52982024-02-13T21:41:26.328Z26/30 Read block 358 len 8192
52992024-02-13T21:41:26.328Z27/30 Write block 319 len 36864 data: 4 4 4 3 4 3 4 4 5
53002024-02-13T21:41:26.328Z28/30 Write block 152 len 8192 data: 3 4
53012024-02-13T21:41:26.328Z29/30 Write block 172 len 16384 data: 3 3 3 3
53022024-02-13T21:41:26.328Z30/30 Read block 224 len 24576
53032024-02-13T21:41:26.329Z----------------------------------------------------------------
53042024-02-13T21:41:26.329Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
53052024-02-13T21:41:26.329ZStates: Active Active Active
53062024-02-13T21:41:26.329ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53072024-02-13T21:41:26.329Z 1 Acked 1000 Write 7 Done Done Done false
53082024-02-13T21:41:26.329Z 2 Acked 1001 Write 10 Done Sent Done false
53092024-02-13T21:41:26.330Z 3 Acked 1002 Write 10 Done Sent Done false
53102024-02-13T21:41:26.330Z 4 Acked 1003 Write 3 Done Sent Done false
53112024-02-13T21:41:26.330Z 5 Acked 1004 Read 7 Done Sent Done false
53122024-02-13T21:41:26.330Z 6 Acked 1005 Read 6 Done Sent Done false
53132024-02-13T21:41:26.330Z 7 Acked 1006 Write 10 Done Sent Done false
53142024-02-13T21:41:26.330Z 8 Acked 1007 Write 7 Done Sent Done false
53152024-02-13T21:41:26.330Z 9 Acked 1008 Write 1 Done Sent Done false
53162024-02-13T21:41:26.331Z 10 Acked 1009 Read 2 Done Sent Done false
53172024-02-13T21:41:26.331Z 11 Acked 1010 Read 4 Done Sent Done false
53182024-02-13T21:41:26.331Z 12 Acked 1011 Write 5 Done Sent Done false
53192024-02-13T21:41:26.331Z 13 Acked 1012 Write 2 Done Sent Done false
53202024-02-13T21:41:26.331Z 14 Acked 1013 Read 5 Done Sent Done false
53212024-02-13T21:41:26.331Z 15 Acked 1014 Write 5 Done Sent Done false
53222024-02-13T21:41:26.331Z 16 Acked 1015 Read 2 Done Sent Done false
53232024-02-13T21:41:26.331Z 17 Acked 1016 Read 3 Done Sent Done false
53242024-02-13T21:41:26.332Z 18 Acked 1017 Write 4 Done Sent Done false
53252024-02-13T21:41:26.332Z 19 Acked 1018 Write 6 Done Sent Done false
53262024-02-13T21:41:26.332Z 20 Acked 1019 Read 10 Done Sent Done false
53272024-02-13T21:41:26.332Z 21 Acked 1020 Read 2 Done Sent Done false
53282024-02-13T21:41:26.332Z 22 Acked 1021 Write 2 Done Sent Done false
53292024-02-13T21:41:26.332Z 23 Acked 1022 Read 6 Done Sent Done false
53302024-02-13T21:41:26.332Z 24 Acked 1023 Write 4 Done Sent Done false
53312024-02-13T21:41:26.332Z 25 Acked 1024 Write 1 Done Sent Done false
53322024-02-13T21:41:26.332Z 26 Acked 1025 Read 2 Done Sent Done false
53332024-02-13T21:41:26.333Z 27 Acked 1026 Write 9 Done Sent Done false
53342024-02-13T21:41:26.333Z 28 Acked 1027 Write 2 Done Sent Done false
53352024-02-13T21:41:26.333Z 29 Acked 1028 Write 4 Sent Sent Done false
53362024-02-13T21:41:26.333Z 30 Acked 1029 Read 6 Sent Sent Done false
53372024-02-13T21:41:26.333Z STATES DS:0 DS:1 DS:2 TOTAL
53382024-02-13T21:41:26.333Z New 0 0 0 0
53392024-02-13T21:41:26.333Z Sent 2 29 0 31
53402024-02-13T21:41:26.334Z Done 28 1 30 59
53412024-02-13T21:41:26.334Z Skipped 0 0 0 0
53422024-02-13T21:41:26.334Z Error 0 0 0 0
53432024-02-13T21:41:26.334ZLast Flush: 0 0 0
53442024-02-13T21:41:26.334ZDownstairs last five completed:
53452024-02-13T21:41:26.334ZUpstairs last five completed: 30 29 28 27 26
53462024-02-13T21:41:26.334Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:25.123532584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1330}
53472024-02-13T21:41:26.334ZWrote out file "/tmp/test_repair_verify.data"
53482024-02-13T21:41:26.334Z
53492024-02-13T21:41:26.335ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53502024-02-13T21:41:26.343ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53512024-02-13T21:41:26.343Z 0 000-019 42 42 42 65 65 65 F F F
53522024-02-13T21:41:26.343Z 1 020-039 36 36 36 60 60 60 T F T <---
53532024-02-13T21:41:26.343Z 2 040-059 46 46 46 74 74 74 T F T <---
53542024-02-13T21:41:26.344Z 3 060-079 47 47 47 75 75 75 T F T <---
53552024-02-13T21:41:26.344Z 4 080-099 47 47 47 75 75 75 T F T <---
53562024-02-13T21:41:26.344Z 5 100-119 44 44 44 70 70 70 T F T <---
53572024-02-13T21:41:26.344Z 6 120-139 42 42 42 65 65 65 F F F
53582024-02-13T21:41:26.349Z 7 140-159 47 47 47 75 75 75 T F T <---
53592024-02-13T21:41:26.349Z 8 160-179 39 39 39 62 62 62 T F T <---
53602024-02-13T21:41:26.349Z 9 180-199 45 45 45 71 71 71 T F T <---
53612024-02-13T21:41:26.349Z 10 200-219 44 44 44 67 67 67 F F F
53622024-02-13T21:41:26.350Z 11 220-239 45 45 45 71 71 71 F F F
53632024-02-13T21:41:26.350Z 12 240-259 45 45 45 71 71 71 F F F
53642024-02-13T21:41:26.350Z 13 260-279 47 47 47 75 75 75 F F F
53652024-02-13T21:41:26.350Z 14 280-299 47 47 47 75 75 75 F F F
53662024-02-13T21:41:26.350Z 15 300-319 46 46 46 73 73 73 T F T <---
53672024-02-13T21:41:26.350Z 16 320-339 46 46 46 73 73 73 T F T <---
53682024-02-13T21:41:26.350Z 17 340-359 47 47 47 75 75 75 T T T
53692024-02-13T21:41:26.351Z 18 360-379 47 47 47 75 75 75 F F F
53702024-02-13T21:41:26.351Z 19 380-399 45 45 45 71 71 71 F F F
53712024-02-13T21:41:26.351Z 20 400-419 47 47 47 75 75 75 F F F
53722024-02-13T21:41:26.351Z 21 420-439 41 41 41 64 64 64 F F F
53732024-02-13T21:41:26.351Z 22 440-459 47 47 47 75 75 75 F F F
53742024-02-13T21:41:26.351Z 23 460-479 44 44 44 68 68 68 F F F
53752024-02-13T21:41:26.351Z 24 480-499 46 46 46 74 74 74 F F F
53762024-02-13T21:41:26.351Z 25 500-519 47 47 47 75 75 75 F F F
53772024-02-13T21:41:26.351Z 26 520-539 47 47 47 75 75 75 T F T <---
53782024-02-13T21:41:26.351Z 27 540-559 43 43 43 66 66 66 T F T <---
53792024-02-13T21:41:26.351Z 28 560-579 44 44 44 67 67 67 F F F
53802024-02-13T21:41:26.352Z 29 580-599 47 47 47 75 75 75 F F F
53812024-02-13T21:41:26.352ZMax gen: 47, Max flush: 75
53822024-02-13T21:41:26.352ZError: Difference in extent metadata found!
53832024-02-13T21:41:26.352ZA Difference in extent metadata is expected here
53842024-02-13T21:41:26.352ZOn loop 24
53852024-02-13T21:41:26.352Z
53862024-02-13T21:41:26.359ZVerifying data now
53872024-02-13T21:41:31.012ZNo change after verify
53882024-02-13T21:41:31.012ZLoop: 24 Downstairs dump after verify (and repair):
53892024-02-13T21:41:31.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53902024-02-13T21:41:31.027Z 0 000-019 42 42 42 65 65 65 F F F
53912024-02-13T21:41:31.027Z 1 020-039 48 48 48 76 76 76 F F F
53922024-02-13T21:41:31.027Z 2 040-059 48 48 48 76 76 76 F F F
53932024-02-13T21:41:31.027Z 3 060-079 48 48 48 76 76 76 F F F
53942024-02-13T21:41:31.027Z 4 080-099 48 48 48 76 76 76 F F F
53952024-02-13T21:41:31.033Z 5 100-119 48 48 48 76 76 76 F F F
53962024-02-13T21:41:31.034Z 6 120-139 42 42 42 65 65 65 F F F
53972024-02-13T21:41:31.034Z 7 140-159 48 48 48 76 76 76 F F F
53982024-02-13T21:41:31.034Z 8 160-179 48 48 48 76 76 76 F F F
53992024-02-13T21:41:31.034Z 9 180-199 48 48 48 76 76 76 F F F
54002024-02-13T21:41:31.034Z 10 200-219 44 44 44 67 67 67 F F F
54012024-02-13T21:41:31.034Z 11 220-239 45 45 45 71 71 71 F F F
54022024-02-13T21:41:31.034Z 12 240-259 45 45 45 71 71 71 F F F
54032024-02-13T21:41:31.034Z 13 260-279 47 47 47 75 75 75 F F F
54042024-02-13T21:41:31.035Z 14 280-299 47 47 47 75 75 75 F F F
54052024-02-13T21:41:31.035Z 15 300-319 48 48 48 76 76 76 F F F
54062024-02-13T21:41:31.035Z 16 320-339 48 48 48 76 76 76 F F F
54072024-02-13T21:41:31.035Z 17 340-359 48 48 48 76 76 76 F F F
54082024-02-13T21:41:31.035Z 18 360-379 47 47 47 75 75 75 F F F
54092024-02-13T21:41:31.035Z 19 380-399 45 45 45 71 71 71 F F F
54102024-02-13T21:41:31.035Z 20 400-419 47 47 47 75 75 75 F F F
54112024-02-13T21:41:31.035Z 21 420-439 41 41 41 64 64 64 F F F
54122024-02-13T21:41:31.035Z 22 440-459 47 47 47 75 75 75 F F F
54132024-02-13T21:41:31.036Z 23 460-479 44 44 44 68 68 68 F F F
54142024-02-13T21:41:31.036Z 24 480-499 46 46 46 74 74 74 F F F
54152024-02-13T21:41:31.036Z 25 500-519 47 47 47 75 75 75 F F F
54162024-02-13T21:41:31.036Z 26 520-539 48 48 48 76 76 76 F F F
54172024-02-13T21:41:31.036Z 27 540-559 48 48 48 76 76 76 F F F
54182024-02-13T21:41:31.036Z 28 560-579 44 44 44 67 67 67 F F F
54192024-02-13T21:41:31.036Z 29 580-599 47 47 47 75 75 75 F F F
54202024-02-13T21:41:31.037ZMax gen: 48, Max flush: 76
54212024-02-13T21:41:31.037Z
54222024-02-13T21:41:31.037ZBegin loop 25
54232024-02-13T21:41:31.037ZDownstairs to restart: 1
54242024-02-13T21:41:31.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.850767494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
54252024-02-13T21:41:31.055Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.85102826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
54262024-02-13T21:41:31.055Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851067316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
54272024-02-13T21:41:31.055Z{"msg":"Crucible aa9d1a09-69a1-46d1-b513-ed2024f8a44a has session id: 3ce6fc5a-27f6-4284-a6de-559dbc89e16b","v":0,"name":"Crucible runtime is spawned
54282024-02-13T21:41:31.055Zcrucible","level":30,"time":"2024-02-13T21:41:29.851109281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54292024-02-13T21:41:31.056Z{"msg":"Upstairs opts: Upstairs UUID: aa9d1a09-69a1-46d1-b513-ed2024f8a44a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851154344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54302024-02-13T21:41:31.056Z{"msg":"Crucible stats registered with UUID: aa9d1a09-69a1-46d1-b513-ed2024f8a44a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851190641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54312024-02-13T21:41:31.056Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851231046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
54322024-02-13T21:41:31.056Z{"msg":"aa9d1a09-69a1-46d1-b513-ed2024f8a44a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851362319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54332024-02-13T21:41:31.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851398656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54342024-02-13T21:41:31.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851416839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54352024-02-13T21:41:31.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.85144089Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54362024-02-13T21:41:31.057Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851485454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54372024-02-13T21:41:31.057Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851515633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54382024-02-13T21:41:31.057Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851539085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54392024-02-13T21:41:31.057Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851575112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54402024-02-13T21:41:31.057Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851619705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54412024-02-13T21:41:31.057Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851646846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54422024-02-13T21:41:31.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851672276Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54432024-02-13T21:41:31.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851695108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54442024-02-13T21:41:31.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.851789224Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"io task","client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54452024-02-13T21:41:31.058Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.852391385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54462024-02-13T21:41:31.058Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.852427942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54472024-02-13T21:41:31.058Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.852443717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54482024-02-13T21:41:31.058Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.852466298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54492024-02-13T21:41:31.058Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853021487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54502024-02-13T21:41:31.058Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853142343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54512024-02-13T21:41:31.059Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853433518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54522024-02-13T21:41:31.059Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853483499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54532024-02-13T21:41:31.059Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853517447Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54542024-02-13T21:41:31.059Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853548985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54552024-02-13T21:41:31.059Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.85356424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54562024-02-13T21:41:31.059Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853641772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54572024-02-13T21:41:31.059Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853669302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54582024-02-13T21:41:31.059Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.853683567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54592024-02-13T21:41:31.060Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854135443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54602024-02-13T21:41:31.060Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854534518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54612024-02-13T21:41:31.060Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854572444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54622024-02-13T21:41:31.060Z{"msg":"[0]R flush_numbers[0..12]: [65, 76, 76, 76, 76, 76, 65, 76, 76, 76, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854589748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54632024-02-13T21:41:31.060Z{"msg":"[0]R generation[0..12]: [42, 48, 48, 48, 48, 48, 42, 48, 48, 48, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854605013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54642024-02-13T21:41:31.060Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854626455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54652024-02-13T21:41:31.061Z{"msg":"[1]R flush_numbers[0..12]: [65, 76, 76, 76, 76, 76, 65, 76, 76, 76, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854640789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54662024-02-13T21:41:31.061Z{"msg":"[1]R generation[0..12]: [42, 48, 48, 48, 48, 48, 42, 48, 48, 48, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854655314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54672024-02-13T21:41:31.061Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854676047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54682024-02-13T21:41:31.061Z{"msg":"[2]R flush_numbers[0..12]: [65, 76, 76, 76, 76, 76, 65, 76, 76, 76, 67, 71]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854690142Z","hostname":"Wait for a query_work_queue command to finish before sending IO
54692024-02-13T21:41:31.061Zip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54702024-02-13T21:41:31.061Z{"msg":"[2]R generation[0..12]: [42, 48, 48, 48, 48, 48, 42, 48, 48, 48, 44, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854718301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54712024-02-13T21:41:31.062Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854732836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54722024-02-13T21:41:31.062Z{"msg":"Max found gen is 49","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854752709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54732024-02-13T21:41:31.062Z{"msg":"Generation requested: 50 >= found:49","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.85477833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54742024-02-13T21:41:31.062Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854794484Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54752024-02-13T21:41:31.062Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854820414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"":"downstairs","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54762024-02-13T21:41:31.062Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854834619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54772024-02-13T21:41:31.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854853542Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"0","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54782024-02-13T21:41:31.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854878633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"1","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54792024-02-13T21:41:31.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854892828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"client":"2","":"downstairs","session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54802024-02-13T21:41:31.063Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854906733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54812024-02-13T21:41:31.063Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854926406Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54822024-02-13T21:41:31.063Z{"msg":"aa9d1a09-69a1-46d1-b513-ed2024f8a44a is now active with session: 3ce6fc5a-27f6-4284-a6de-559dbc89e16b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854939711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54832024-02-13T21:41:31.063Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.854957914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338,"session_id":"3ce6fc5a-27f6-4284-a6de-559dbc89e16b"}
54842024-02-13T21:41:31.063Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:29.85497072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
54852024-02-13T21:41:31.063ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54862024-02-13T21:41:31.068ZLoading write count information from file "/tmp/test_repair_verify.data"
54872024-02-13T21:41:31.068ZRun Repair workload
54882024-02-13T21:41:31.068Z01/30 Write block 473 len 28672 data: 8 8 7 4 4 5 5
54892024-02-13T21:41:31.068Z02/30 Write block 431 len 4096 data: 5
54902024-02-13T21:41:31.069Z03/30 Flush
54912024-02-13T21:41:31.069Z04/30 Flush
54922024-02-13T21:41:31.069Z05/30 Write block 171 len 12288 data: 2 4 4
54932024-02-13T21:41:31.069Z06/30 Read block 467 len 16384
54942024-02-13T21:41:31.069Z07/30 Read block 537 len 36864
54952024-02-13T21:41:31.069Z08/30 Write block 429 len 24576 data: 6 6 6 6 4 5
54962024-02-13T21:41:31.069Z09/30 Write block 221 len 20480 data: 5 6 6 6 5
54972024-02-13T21:41:31.070Z10/30 Read block 155 len 32768
54982024-02-13T21:41:31.070Z11/30 Flush
54992024-02-13T21:41:31.070Z12/30 Write block 242 len 40960 data: 7 6 3 3 3 3 2 3 3 4
55002024-02-13T21:41:31.070Z13/30 Read block 193 len 36864
55012024-02-13T21:41:31.070Z14/30 Write block 431 len 36864 data: 7 7 5 6 5 3 2 3 3
55022024-02-13T21:41:31.079Z15/30 Read block 97 len 36864
55032024-02-13T21:41:31.079Z16/30 Read block 421 len 16384
55042024-02-13T21:41:31.079Z17/30 Flush
55052024-02-13T21:41:31.079Z18/30 Write block 102 len 32768 data: 8 8 7 6 4 3 3 3
55062024-02-13T21:41:31.080Z19/30 Read block 523 len 12288
55072024-02-13T21:41:31.080Z20/30 Write block 492 len 12288 data: 3 5 8
55082024-02-13T21:41:31.080Z21/30 Read block 24 len 8192
55092024-02-13T21:41:31.080Z22/30 Write block 85 len 32768 data: 6 5 4 5 4 3 3 3
55102024-02-13T21:41:31.080Z23/30 Read block 500 len 32768
55112024-02-13T21:41:31.504Z24/30 Flush
55122024-02-13T21:41:31.504Z25/30 Write block 244 len 20480 data: 4 4 4 4 3
55132024-02-13T21:41:31.504Z26/30 Read block 278 len 28672
55142024-02-13T21:41:31.504Z27/30 Write block 512 len 36864 data: 7 8 9 8 7 9 12 14 15
55152024-02-13T21:41:31.505Z28/30 Read block 34 len 24576
55162024-02-13T21:41:31.505Z29/30 Write block 133 len 4096 data: 5
55172024-02-13T21:41:31.505Z30/30 Write block 31 len 12288 data: 6 8 7
55182024-02-13T21:41:31.505Z----------------------------------------------------------------
55192024-02-13T21:41:31.505Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
55202024-02-13T21:41:31.505ZStates: Active Active Active
55212024-02-13T21:41:31.506ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55222024-02-13T21:41:31.506Z 1 Acked 1000 Write 7 Done Sent Done false
55232024-02-13T21:41:31.506Z 2 Acked 1001 Write 1 Done Sent Done false
55242024-02-13T21:41:31.506Z 3 Acked 1002 Flush 0 Done Sent Done false
55252024-02-13T21:41:31.506Z 4 Acked 1003 Flush 0 Done Sent Done false
55262024-02-13T21:41:31.506Z 5 Acked 1004 Write 3 Done Sent Done false
55272024-02-13T21:41:31.506Z 6 Acked 1005 Read 4 Done Sent Done false
55282024-02-13T21:41:31.506Z 7 Acked 1006 Read 9 Done Sent Done false
55292024-02-13T21:41:31.507Z 8 Acked 1007 Write 6 Done Sent Done false
55302024-02-13T21:41:31.507Z 9 Acked 1008 Write 5 Done Sent Done false
55312024-02-13T21:41:31.507Z 10 Acked 1009 Read 8 Done Sent Done false
55322024-02-13T21:41:31.508Z 11 Acked 1010 Flush 0 Done Sent Done false
55332024-02-13T21:41:31.508Z 12 Acked 1011 Write 10 Done Sent Done false
55342024-02-13T21:41:31.508Z 13 Acked 1012 Read 9 Done Sent Done false
55352024-02-13T21:41:31.508Z 14 Acked 1013 Write 9 Done Sent Done false
55362024-02-13T21:41:31.508Z 15 Acked 1014 Read 9 Done Sent Done false
55372024-02-13T21:41:31.508Z 16 Acked 1015 Read 4 Done Sent Done false
55382024-02-13T21:41:31.508Z 17 Acked 1016 Flush 0 Done Sent Done false
55392024-02-13T21:41:31.508Z 18 Acked 1017 Write 8 Done Sent Done false
55402024-02-13T21:41:31.509Z 19 Acked 1018 Read 3 Done Sent Done false
55412024-02-13T21:41:31.509Z 20 Acked 1019 Write 3 Done Sent Done false
55422024-02-13T21:41:31.509Z 21 Acked 1020 Read 2 Done Sent Done false
55432024-02-13T21:41:31.509Z 22 Acked 1021 Write 8 Done Sent Done false
55442024-02-13T21:41:31.509Z 23 Acked 1022 Read 8 Done Sent Done false
55452024-02-13T21:41:31.509Z 24 Acked 1023 Flush 0 Done Sent Done false
55462024-02-13T21:41:31.509Z 25 Acked 1024 Write 5 Done Sent Done false
55472024-02-13T21:41:31.510Z 26 Acked 1025 Read 7 Done Sent Done false
55482024-02-13T21:41:31.510Z 27 Acked 1026 Write 9 Done Sent Done false
55492024-02-13T21:41:31.510Z 28 Acked 1027 Read 6 Done Sent Done false
55502024-02-13T21:41:31.510Z 29 Acked 1028 Write 1 Done Sent Done false
55512024-02-13T21:41:31.510Z 30 Acked 1029 Write 3 Sent Sent Sent false
55522024-02-13T21:41:31.510Z STATES DS:0 DS:1 DS:2 TOTAL
55532024-02-13T21:41:31.511Z New 0 0 0 0
55542024-02-13T21:41:31.511Z Sent 1 30 1 32
55552024-02-13T21:41:31.511Z Done 29 0 29 58
55562024-02-13T21:41:31.511Z Skipped 0 0 0 0
55572024-02-13T21:41:31.511Z Error 0 0 0 0
55582024-02-13T21:41:31.511ZLast Flush: 1023 0 1023
55592024-02-13T21:41:31.511ZDownstairs last five completed:
55602024-02-13T21:41:31.511ZUpstairs last five completed: 30 29 28 27 26
55612024-02-13T21:41:31.512Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:29.889604917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1338}
55622024-02-13T21:41:31.512ZWrote out file "/tmp/test_repair_verify.data"
55632024-02-13T21:41:31.512Z
55642024-02-13T21:41:31.528ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55652024-02-13T21:41:31.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55662024-02-13T21:41:31.528Z 0 000-019 42 42 42 65 65 65 F F F
55672024-02-13T21:41:31.528Z 1 020-039 48 48 48 76 76 76 T F T <---
55682024-02-13T21:41:31.529Z 2 040-059 48 48 48 76 76 76 F F F
55692024-02-13T21:41:31.529Z 3 060-079 48 48 48 76 76 76 F F F
55702024-02-13T21:41:31.529Z 4 080-099 50 48 50 81 76 81 F F F <---
55712024-02-13T21:41:31.529Z 5 100-119 50 48 50 81 76 81 F F F <---
55722024-02-13T21:41:31.529Z 6 120-139 42 42 42 65 65 65 T F T <---
55732024-02-13T21:41:31.529Z 7 140-159 48 48 48 76 76 76 F F F
55742024-02-13T21:41:31.529Z 8 160-179 50 48 50 79 76 79 F F F <---
55752024-02-13T21:41:31.529Z 9 180-199 48 48 48 76 76 76 F F F
55762024-02-13T21:41:31.529Z 10 200-219 44 44 44 67 67 67 F F F
55772024-02-13T21:41:31.530Z 11 220-239 50 45 50 79 71 79 F F F <---
55782024-02-13T21:41:31.530Z 12 240-259 50 45 50 80 71 80 T F T <---
55792024-02-13T21:41:31.530Z 13 260-279 47 47 47 75 75 75 F F F
55802024-02-13T21:41:31.530Z 14 280-299 47 47 47 75 75 75 F F F
55812024-02-13T21:41:31.530Z 15 300-319 48 48 48 76 76 76 F F F
55822024-02-13T21:41:31.530Z 16 320-339 48 48 48 76 76 76 F F F
55832024-02-13T21:41:31.530Z 17 340-359 48 48 48 76 76 76 F F F
55842024-02-13T21:41:31.530Z 18 360-379 47 47 47 75 75 75 F F F
55852024-02-13T21:41:31.530Z 19 380-399 45 45 45 71 71 71 F F F
55862024-02-13T21:41:31.531Z 20 400-419 47 47 47 75 75 75 F F F
55872024-02-13T21:41:31.531Z 21 420-439 50 41 50 80 64 80 F F F <---
55882024-02-13T21:41:31.531Z 22 440-459 47 47 47 75 75 75 F F F
55892024-02-13T21:41:31.531Z 23 460-479 50 44 50 77 68 77 F F F <---
55902024-02-13T21:41:31.531Z 24 480-499 50 46 50 81 74 81 F F F <---
55912024-02-13T21:41:31.531Z 25 500-519 47 47 47 75 75 75 T F T <---
55922024-02-13T21:41:31.532Z 26 520-539 48 48 48 76 76 76 T F T <---
55932024-02-13T21:41:31.532Z 27 540-559 48 48 48 76 76 76 F F F
55942024-02-13T21:41:31.532Z 28 560-579 44 44 44 67 67 67 F F F
55952024-02-13T21:41:31.532Z 29 580-599 47 47 47 75 75 75 F F F
55962024-02-13T21:41:31.532ZMax gen: 50, Max flush: 81
55972024-02-13T21:41:31.533ZError: Difference in extent metadata found!
55982024-02-13T21:41:31.533ZA Difference in extent metadata is expected here
55992024-02-13T21:41:31.533ZOn loop 25
56002024-02-13T21:41:31.533Z
56012024-02-13T21:41:31.533ZVerifying data now
56022024-02-13T21:41:35.710ZNo change after verify
56032024-02-13T21:41:35.716ZLoop: 25 Downstairs dump after verify (and repair):
56042024-02-13T21:41:35.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56052024-02-13T21:41:35.725Z 0 000-019 42 42 42 65 65 65 F F F
56062024-02-13T21:41:35.725Z 1 020-039 51 51 51 82 82 82 F F F
56072024-02-13T21:41:35.725Z 2 040-059 48 48 48 76 76 76 F F F
56082024-02-13T21:41:35.726Z 3 060-079 48 48 48 76 76 76 F F F
56092024-02-13T21:41:35.726Z 4 080-099 50 50 50 81 81 81 F F F
56102024-02-13T21:41:35.726Z 5 100-119 50 50 50 81 81 81 F F F
56112024-02-13T21:41:35.736Z 6 120-139 51 51 51 82 82 82 F F F
56122024-02-13T21:41:35.736Z 7 140-159 48 48 48 76 76 76 F F F
56132024-02-13T21:41:35.736Z 8 160-179 50 50 50 79 79 79 F F F
56142024-02-13T21:41:35.736Z 9 180-199 48 48 48 76 76 76 F F F
56152024-02-13T21:41:35.737Z 10 200-219 44 44 44 67 67 67 F F F
56162024-02-13T21:41:35.737Z 11 220-239 50 50 50 79 79 79 F F F
56172024-02-13T21:41:35.737Z 12 240-259 51 51 51 82 82 82 F F F
56182024-02-13T21:41:35.737Z 13 260-279 47 47 47 75 75 75 F F F
56192024-02-13T21:41:35.737Z 14 280-299 47 47 47 75 75 75 F F F
56202024-02-13T21:41:35.737Z 15 300-319 48 48 48 76 76 76 F F F
56212024-02-13T21:41:35.737Z 16 320-339 48 48 48 76 76 76 F F F
56222024-02-13T21:41:35.738Z 17 340-359 48 48 48 76 76 76 F F F
56232024-02-13T21:41:35.738Z 18 360-379 47 47 47 75 75 75 F F F
56242024-02-13T21:41:35.738Z 19 380-399 45 45 45 71 71 71 F F F
56252024-02-13T21:41:35.738Z 20 400-419 47 47 47 75 75 75 F F F
56262024-02-13T21:41:35.738Z 21 420-439 50 50 50 80 80 80 F F F
56272024-02-13T21:41:35.738Z 22 440-459 47 47 47 75 75 75 F F F
56282024-02-13T21:41:35.738Z 23 460-479 50 50 50 77 77 77 F F F
56292024-02-13T21:41:35.739Z 24 480-499 50 50 50 81 81 81 F F F
56302024-02-13T21:41:35.739Z 25 500-519 51 51 51 82 82 82 F F F
56312024-02-13T21:41:35.739Z 26 520-539 51 51 51 82 82 82 F F F
56322024-02-13T21:41:35.739Z 27 540-559 48 48 48 76 76 76 F F F
56332024-02-13T21:41:35.739Z 28 560-579 44 44 44 67 67 67 F F F
56342024-02-13T21:41:35.739Z 29 580-599 47 47 47 75 75 75 F F F
56352024-02-13T21:41:35.739ZMax gen: 51, Max flush: 82
56362024-02-13T21:41:35.739Z
56372024-02-13T21:41:35.740ZBegin loop 26
56382024-02-13T21:41:35.740ZDownstairs to restart: 1
56392024-02-13T21:41:35.750Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549274497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
56402024-02-13T21:41:35.750Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
56412024-02-13T21:41:35.760Z rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549611275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
56422024-02-13T21:41:35.760Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549659877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
56432024-02-13T21:41:35.760Z{"msg":"Crucible 101820f9-a287-4b24-86d2-24964e944d2a has session id: 8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549689396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56442024-02-13T21:41:35.761Z{"msg":"Upstairs opts: Upstairs UUID: 101820f9-a287-4b24-86d2-24964e944d2a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549744117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56452024-02-13T21:41:35.761Z{"msg":"Crucible stats registered with UUID: 101820f9-a287-4b24-86d2-24964e944d2a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549785631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56462024-02-13T21:41:35.761Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549814421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
56472024-02-13T21:41:35.761Z{"msg":"101820f9-a287-4b24-86d2-24964e944d2a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549856316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56482024-02-13T21:41:35.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549884986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56492024-02-13T21:41:35.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549915894Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56502024-02-13T21:41:35.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549945833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56512024-02-13T21:41:35.762Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549976002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56522024-02-13T21:41:35.762Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.549992196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56532024-02-13T21:41:35.762Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550023875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56542024-02-13T21:41:35.762Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550042508Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56552024-02-13T21:41:35.762Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550076236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56562024-02-13T21:41:35.762Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550104136Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56572024-02-13T21:41:35.762Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550195603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56582024-02-13T21:41:35.762Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.55022943Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56592024-02-13T21:41:35.762Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.55025773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"io task","client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56602024-02-13T21:41:35.762Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.550979189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56612024-02-13T21:41:35.762Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.551025702Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56622024-02-13T21:41:35.762Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.55105676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56632024-02-13T21:41:35.763Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.551098245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56642024-02-13T21:41:35.763Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.551649865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56652024-02-13T21:41:35.763Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.551833968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56662024-02-13T21:41:35.763Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552095593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56672024-02-13T21:41:35.763Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552159121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56682024-02-13T21:41:35.763Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552208503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56692024-02-13T21:41:35.763Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552245159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56702024-02-13T21:41:35.763Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552274239Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56712024-02-13T21:41:35.763Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56722024-02-13T21:41:35.763Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552320302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56732024-02-13T21:41:35.763Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.55235306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56742024-02-13T21:41:35.763Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.552661838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56752024-02-13T21:41:35.763Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553054686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56762024-02-13T21:41:35.763Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553094261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56772024-02-13T21:41:35.763Z{"msg":"[0]R flush_numbers[0..12]: [65, 82, 76, 76, 81, 81, 82, 76, 79, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553111495Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56782024-02-13T21:41:35.763Z{"msg":"[0]R generation[0..12]: [42, 51, 48, 48, 50, 50, 51, 48, 50, 48, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553133697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56792024-02-13T21:41:35.763Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553148902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56802024-02-13T21:41:35.763Z{"msg":"[1]R flush_numbers[0..12]: [65, 82, 76, 76, 81, 81, 82, 76, 79, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553172203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56812024-02-13T21:41:35.764Z{"msg":"[1]R generation[0..12]: [42, 51, 48, 48, 50, 50, 51, 48, 50, 48, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553186578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56822024-02-13T21:41:35.764Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.55320751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56832024-02-13T21:41:35.764Z{"msg":"[2]R flush_numbers[0..12]: [65, 82, 76, 76, 81, 81, 82, 76, 79, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553221915Z","Wait for a query_work_queue command to finish before sending IO
56842024-02-13T21:41:35.764Zhostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56852024-02-13T21:41:35.764Z{"msg":"[2]R generation[0..12]: [42, 51, 48, 48, 50, 50, 51, 48, 50, 48, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553247246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56862024-02-13T21:41:35.764Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553261561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56872024-02-13T21:41:35.764Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553281493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56882024-02-13T21:41:35.764Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553296168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56892024-02-13T21:41:35.764Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553320739Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56902024-02-13T21:41:35.764Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553334164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"":"downstairs","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56912024-02-13T21:41:35.764Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553354717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56922024-02-13T21:41:35.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553367122Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"0","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56932024-02-13T21:41:35.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553380418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"1","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56942024-02-13T21:41:35.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553393493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"client":"2","":"downstairs","session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56952024-02-13T21:41:35.764Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553406648Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56962024-02-13T21:41:35.764Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553419013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56972024-02-13T21:41:35.764Z{"msg":"101820f9-a287-4b24-86d2-24964e944d2a is now active with session: 8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553438616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56982024-02-13T21:41:35.764Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553450792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346,"session_id":"8ffa1a6f-f7cb-4b20-b9f6-351b0c74060e"}
56992024-02-13T21:41:35.764Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:34.553462977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
57002024-02-13T21:41:35.764ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57012024-02-13T21:41:35.765ZLoading write count information from file "/tmp/test_repair_verify.data"
57022024-02-13T21:41:35.765ZRun Repair workload
57032024-02-13T21:41:35.765Z01/30 Write block 4 len 32768 data: 4 5 5 5 4 5 5 7
57042024-02-13T21:41:35.765Z02/30 Write block 345 len 36864 data: 4 4 6 5 3 3 5 6 6
57052024-02-13T21:41:35.765Z03/30 Read block 230 len 4096
57062024-02-13T21:41:35.769Z04/30 Write block 403 len 40960 data: 3 3 3 2 3 3 3 3 4 4
57072024-02-13T21:41:35.769Z05/30 Read block 275 len 36864
57082024-02-13T21:41:35.770Z06/30 Read block 122 len 28672
57092024-02-13T21:41:35.770Z07/30 Read block 52 len 32768
57102024-02-13T21:41:35.770Z08/30 Write block 409 len 12288 data: 4 4 5
57112024-02-13T21:41:35.770Z09/30 Write block 2 len 16384 data: 4 4 5 6
57122024-02-13T21:41:35.770Z10/30 Write block 68 len 40960 data: 4 4 5 5 6 4 5 5 5 5
57132024-02-13T21:41:35.770Z11/30 Flush
57142024-02-13T21:41:35.770Z12/30 Write block 400 len 12288 data: 3 3 3
57152024-02-13T21:41:35.770Z13/30 Read block 458 len 36864
57162024-02-13T21:41:35.771Z14/30 Read block 175 len 4096
57172024-02-13T21:41:35.771Z15/30 Read block 70 len 28672
57182024-02-13T21:41:35.771Z16/30 Read block 575 len 32768
57192024-02-13T21:41:35.775Z17/30 Write block 169 len 40960 data: 2 2 3 5 5 4 4 3 5 4
57202024-02-13T21:41:35.775Z18/30 Write block 353 len 32768 data: 7 6 7 8 8 7 7 6
57212024-02-13T21:41:35.776Z19/30 Read block 257 len 40960
57222024-02-13T21:41:35.776Z20/30 Write block 322 len 24576 data: 4 5 4 5 5 6
57232024-02-13T21:41:35.776Z21/30 Read block 264 len 40960
57242024-02-13T21:41:35.776Z22/30 Read block 490 len 16384
57252024-02-13T21:41:35.781Z23/30 Read block 81 len 12288
57262024-02-13T21:41:35.781Z24/30 Read block 533 len 4096
57272024-02-13T21:41:35.781Z25/30 Write block 363 len 8192 data: 3 3
57282024-02-13T21:41:35.781Z26/30 Write block 504 len 32768 data: 3 3 5 5 5 6 7 7
57292024-02-13T21:41:35.781Z27/30 Write block 131 len 24576 data: 6 6 6 4 5 4
57302024-02-13T21:41:35.782Z28/30 Read block 413 len 32768
57312024-02-13T21:41:35.787Z29/30 Read block 25 len 32768
57322024-02-13T21:41:35.787Z30/30 Write block 589 len 8192 data: 4 5
57332024-02-13T21:41:35.787Z----------------------------------------------------------------
57342024-02-13T21:41:35.787Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57352024-02-13T21:41:35.787ZStates: Active Active Active
57362024-02-13T21:41:35.787ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57372024-02-13T21:41:35.788Z 1 Acked 1000 Write 8 Done Sent Done false
57382024-02-13T21:41:35.788Z 2 Acked 1001 Write 9 Done Sent Done false
57392024-02-13T21:41:35.788Z 3 Acked 1002 Read 1 Done Sent Done false
57402024-02-13T21:41:35.788Z 4 Acked 1003 Write 10 Done Sent Done false
57412024-02-13T21:41:35.788Z 5 Acked 1004 Read 9 Done Sent Done false
57422024-02-13T21:41:35.789Z 6 Acked 1005 Read 7 Done Sent Done false
57432024-02-13T21:41:35.789Z 7 Acked 1006 Read 8 Done Sent Done false
57442024-02-13T21:41:35.789Z 8 Acked 1007 Write 3 Done Sent Done false
57452024-02-13T21:41:35.789Z 9 Acked 1008 Write 4 Done Sent Done false
57462024-02-13T21:41:35.789Z 10 Acked 1009 Write 10 Done Sent Done false
57472024-02-13T21:41:35.789Z 11 Acked 1010 Flush 0 Done Sent Done false
57482024-02-13T21:41:35.789Z 12 Acked 1011 Write 3 Done Sent Done false
57492024-02-13T21:41:35.790Z 13 Acked 1012 Read 9 Done Sent Done false
57502024-02-13T21:41:35.790Z 14 Acked 1013 Read 1 Done Sent Done false
57512024-02-13T21:41:35.790Z 15 Acked 1014 Read 7 Done Sent Done false
57522024-02-13T21:41:35.790Z 16 Acked 1015 Read 8 Done Sent Done false
57532024-02-13T21:41:35.790Z 17 Acked 1016 Write 10 Done Sent Done false
57542024-02-13T21:41:35.790Z 18 Acked 1017 Write 8 Done Sent Done false
57552024-02-13T21:41:35.790Z 19 Acked 1018 Read 10 Done Sent Done false
57562024-02-13T21:41:35.790Z 20 Acked 1019 Write 6 Done Sent Done false
57572024-02-13T21:41:35.790Z 21 Acked 1020 Read 10 Done Sent Done false
57582024-02-13T21:41:35.790Z 22 Acked 1021 Read 4 Done Sent Done false
57592024-02-13T21:41:35.790Z 23 Acked 1022 Read 3 Done Sent Done false
57602024-02-13T21:41:35.790Z 24 Acked 1023 Read 1 Done Sent Done false
57612024-02-13T21:41:35.790Z 25 Acked 1024 Write 2 Done Sent Done false
57622024-02-13T21:41:35.790Z 26 Acked 1025 Write 8 Done Sent Done false
57632024-02-13T21:41:35.790Z 27 Acked 1026 Write 6 Done Sent Done false
57642024-02-13T21:41:35.791Z 28 Acked 1027 Read 8 Done Sent Done false
57652024-02-13T21:41:35.791Z 29 Acked 1028 Read 8 Done Sent Done false
57662024-02-13T21:41:35.791Z 30 Acked 1029 Write 2 Sent Sent Sent false
57672024-02-13T21:41:35.791Z STATES DS:0 DS:1 DS:2 TOTAL
57682024-02-13T21:41:35.791Z New 0 0 0 0
57692024-02-13T21:41:35.791Z Sent 1 30 1 32
57702024-02-13T21:41:35.791Z Done 29 0 29 58
57712024-02-13T21:41:35.791Z Skipped 0 0 0 0
57722024-02-13T21:41:35.791Z Error 0 0 0 0
57732024-02-13T21:41:35.791ZLast Flush: 1010 0 1010
57742024-02-13T21:41:35.791ZDownstairs last five completed:
57752024-02-13T21:41:35.791ZUpstairs last five completed: 30 29 28 27 26
57762024-02-13T21:41:35.791Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:34.584891806Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1346}
57772024-02-13T21:41:35.791ZWrote out file "/tmp/test_repair_verify.data"
57782024-02-13T21:41:35.797Z
57792024-02-13T21:41:35.797ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57802024-02-13T21:41:35.807ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57812024-02-13T21:41:35.807Z 0 000-019 52 42 52 83 65 83 F F F <---
57822024-02-13T21:41:35.807Z 1 020-039 51 51 51 82 82 82 F F F
57832024-02-13T21:41:35.808Z 2 040-059 48 48 48 76 76 76 F F F
57842024-02-13T21:41:35.808Z 3 060-079 52 48 52 83 76 83 F F F <---
57852024-02-13T21:41:35.808Z 4 080-099 50 50 50 81 81 81 F F F
57862024-02-13T21:41:35.812Z 5 100-119 50 50 50 81 81 81 F F F
57872024-02-13T21:41:35.812Z 6 120-139 51 51 51 82 82 82 T F T <---
57882024-02-13T21:41:35.812Z 7 140-159 48 48 48 76 76 76 F F F
57892024-02-13T21:41:35.813Z 8 160-179 50 50 50 79 79 79 T F T <---
57902024-02-13T21:41:35.813Z 9 180-199 48 48 48 76 76 76 F F F
57912024-02-13T21:41:35.813Z 10 200-219 44 44 44 67 67 67 F F F
57922024-02-13T21:41:35.813Z 11 220-239 50 50 50 79 79 79 F F F
57932024-02-13T21:41:35.813Z 12 240-259 51 51 51 82 82 82 F F F
57942024-02-13T21:41:35.813Z 13 260-279 47 47 47 75 75 75 F F F
57952024-02-13T21:41:35.814Z 14 280-299 47 47 47 75 75 75 F F F
57962024-02-13T21:41:35.814Z 15 300-319 48 48 48 76 76 76 F F F
57972024-02-13T21:41:35.814Z 16 320-339 48 48 48 76 76 76 T F T <---
57982024-02-13T21:41:35.814Z 17 340-359 52 48 52 83 76 83 T F T <---
57992024-02-13T21:41:35.814Z 18 360-379 47 47 47 75 75 75 T F T <---
58002024-02-13T21:41:35.814Z 19 380-399 45 45 45 71 71 71 F F F
58012024-02-13T21:41:35.814Z 20 400-419 52 47 52 83 75 83 T F T <---
58022024-02-13T21:41:35.814Z 21 420-439 50 50 50 80 80 80 F F F
58032024-02-13T21:41:35.814Z 22 440-459 47 47 47 75 75 75 F F F
58042024-02-13T21:41:35.815Z 23 460-479 50 50 50 77 77 77 F F F
58052024-02-13T21:41:35.815Z 24 480-499 50 50 50 81 81 81 F F F
58062024-02-13T21:41:35.815Z 25 500-519 51 51 51 82 82 82 T F T <---
58072024-02-13T21:41:35.815Z 26 520-539 51 51 51 82 82 82 F F F
58082024-02-13T21:41:35.815Z 27 540-559 48 48 48 76 76 76 F F F
58092024-02-13T21:41:35.815Z 28 560-579 44 44 44 67 67 67 F F F
58102024-02-13T21:41:35.815Z 29 580-599 47 47 47 75 75 75 T F T <---
58112024-02-13T21:41:35.816ZMax gen: 52, Max flush: 83
58122024-02-13T21:41:35.816ZError: Difference in extent metadata found!
58132024-02-13T21:41:35.816ZA Difference in extent metadata is expected here
58142024-02-13T21:41:35.816ZOn loop 26
58152024-02-13T21:41:35.816Z
58162024-02-13T21:41:35.821ZVerifying data now
58172024-02-13T21:41:40.338ZNo change after verify
58182024-02-13T21:41:40.343ZLoop: 26 Downstairs dump after verify (and repair):
58192024-02-13T21:41:40.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58202024-02-13T21:41:40.350Z 0 000-019 52 52 52 83 83 83 F F F
58212024-02-13T21:41:40.351Z 1 020-039 51 51 51 82 82 82 F F F
58222024-02-13T21:41:40.351Z 2 040-059 48 48 48 76 76 76 F F F
58232024-02-13T21:41:40.351Z 3 060-079 52 52 52 83 83 83 F F F
58242024-02-13T21:41:40.351Z 4 080-099 50 50 50 81 81 81 F F F
58252024-02-13T21:41:40.357Z 5 100-119 50 50 50 81 81 81 F F F
58262024-02-13T21:41:40.357Z 6 120-139 53 53 53 84 84 84 F F F
58272024-02-13T21:41:40.357Z 7 140-159 48 48 48 76 76 76 F F F
58282024-02-13T21:41:40.357Z 8 160-179 53 53 53 84 84 84 F F F
58292024-02-13T21:41:40.357Z 9 180-199 48 48 48 76 76 76 F F F
58302024-02-13T21:41:40.357Z 10 200-219 44 44 44 67 67 67 F F F
58312024-02-13T21:41:40.358Z 11 220-239 50 50 50 79 79 79 F F F
58322024-02-13T21:41:40.358Z 12 240-259 51 51 51 82 82 82 F F F
58332024-02-13T21:41:40.358Z 13 260-279 47 47 47 75 75 75 F F F
58342024-02-13T21:41:40.358Z 14 280-299 47 47 47 75 75 75 F F F
58352024-02-13T21:41:40.358Z 15 300-319 48 48 48 76 76 76 F F F
58362024-02-13T21:41:40.358Z 16 320-339 53 53 53 84 84 84 F F F
58372024-02-13T21:41:40.359Z 17 340-359 53 53 53 84 84 84 F F F
58382024-02-13T21:41:40.359Z 18 360-379 53 53 53 84 84 84 F F F
58392024-02-13T21:41:40.359Z 19 380-399 45 45 45 71 71 71 F F F
58402024-02-13T21:41:40.359Z 20 400-419 53 53 53 84 84 84 F F F
58412024-02-13T21:41:40.359Z 21 420-439 50 50 50 80 80 80 F F F
58422024-02-13T21:41:40.359Z 22 440-459 47 47 47 75 75 75 F F F
58432024-02-13T21:41:40.359Z 23 460-479 50 50 50 77 77 77 F F F
58442024-02-13T21:41:40.360Z 24 480-499 50 50 50 81 81 81 F F F
58452024-02-13T21:41:40.360Z 25 500-519 53 53 53 84 84 84 F F F
58462024-02-13T21:41:40.360Z 26 520-539 51 51 51 82 82 82 F F F
58472024-02-13T21:41:40.360Z 27 540-559 48 48 48 76 76 76 F F F
58482024-02-13T21:41:40.360Z 28 560-579 44 44 44 67 67 67 F F F
58492024-02-13T21:41:40.361Z 29 580-599 53 53 53 84 84 84 F F F
58502024-02-13T21:41:40.361ZMax gen: 53, Max flush: 84
58512024-02-13T21:41:40.361Z
58522024-02-13T21:41:40.361ZBegin loop 27
58532024-02-13T21:41:40.361ZDownstairs to restart: 1
58542024-02-13T21:41:40.388Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.172492297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
58552024-02-13T21:41:40.396Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.172774865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
58562024-02-13T21:41:40.396Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.172802834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
58572024-02-13T21:41:40.396Z{"msg":"Crucible 57cc6cbe-beba-480c-8a90-9da0a622e663 has session id: 06be693d-7e07-47d0-a3bb-b792ebb38fec","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.172823717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58582024-02-13T21:41:40.396Z{"msg":"Upstairs opts: Upstairs UUID: 57cc6cbe-beba-480c-8a90-9da0a622e663, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.172856145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58592024-02-13T21:41:40.397Z{"msg":"Crucible stats registered with UUID: 57cc6cbe-beba-480c-8a90-9da0a622e663","v":0,"name":"crucible","level":30Crucible runtime is spawned
58602024-02-13T21:41:40.397Z,"time":"2024-02-13T21:41:39.172889783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58612024-02-13T21:41:40.397Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173001023Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
58622024-02-13T21:41:40.397Z{"msg":"57cc6cbe-beba-480c-8a90-9da0a622e663 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173039809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58632024-02-13T21:41:40.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173080854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58642024-02-13T21:41:40.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.1731176Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58652024-02-13T21:41:40.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173142791Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58662024-02-13T21:41:40.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173177669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58672024-02-13T21:41:40.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173203499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58682024-02-13T21:41:40.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173231949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58692024-02-13T21:41:40.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173278292Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58702024-02-13T21:41:40.397Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173310341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58712024-02-13T21:41:40.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173346117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58722024-02-13T21:41:40.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173372328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58732024-02-13T21:41:40.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173406386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58742024-02-13T21:41:40.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.173435585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"io task","client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58752024-02-13T21:41:40.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174063387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58762024-02-13T21:41:40.397Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174104732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58772024-02-13T21:41:40.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174145308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58782024-02-13T21:41:40.397Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174191711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58792024-02-13T21:41:40.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174732445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58802024-02-13T21:41:40.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.174790164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58812024-02-13T21:41:40.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175304527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58822024-02-13T21:41:40.398Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175340044Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58832024-02-13T21:41:40.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175356138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58842024-02-13T21:41:40.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175369124Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58852024-02-13T21:41:40.398Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175383389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58862024-02-13T21:41:40.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.17540578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58872024-02-13T21:41:40.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175433191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58882024-02-13T21:41:40.398Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175448705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58892024-02-13T21:41:40.398Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.175832246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58902024-02-13T21:41:40.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176232041Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58912024-02-13T21:41:40.398Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176270597Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58922024-02-13T21:41:40.398Z{"msg":"[0]R flush_numbers[0..12]: [83, 82, 76, 83, 81, 81, 84, 76, 84, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.1762897Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58932024-02-13T21:41:40.398Z{"msg":"[0]R generation[0..12]: [52, 51, 48, 52, 50, 50, 53, 48, 53, 48, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176321188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58942024-02-13T21:41:40.398Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.17634412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58952024-02-13T21:41:40.398Z{"msg":"[1]R flush_numbers[0..12]: [83, 82, 76, 83, 81, 81, 84, 76, 84, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176373099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58962024-02-13T21:41:40.398Z{"msg":"[1]R generation[0..12]: [52, 51, 48, 52, 50, 50, 53, 48, 53, 48, 44, 50]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
58972024-02-13T21:41:40.398Z,"time":"2024-02-13T21:41:39.176394852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58982024-02-13T21:41:40.399Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176424321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
58992024-02-13T21:41:40.399Z{"msg":"[2]R flush_numbers[0..12]: [83, 82, 76, 83, 81, 81, 84, 76, 84, 76, 67, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.17645564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59002024-02-13T21:41:40.399Z{"msg":"[2]R generation[0..12]: [52, 51, 48, 52, 50, 50, 53, 48, 53, 48, 44, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176486229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59012024-02-13T21:41:40.399Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176519786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59022024-02-13T21:41:40.399Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176535631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59032024-02-13T21:41:40.399Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176567189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59042024-02-13T21:41:40.399Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.17659163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59052024-02-13T21:41:40.399Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176626558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"":"downstairs","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59062024-02-13T21:41:40.399Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176655747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59072024-02-13T21:41:40.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176668942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"0","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59082024-02-13T21:41:40.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176685676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"1","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59092024-02-13T21:41:40.399Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.17672935Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"client":"2","":"downstairs","session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59102024-02-13T21:41:40.399Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176764688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59112024-02-13T21:41:40.399Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176779332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59122024-02-13T21:41:40.399Z{"msg":"57cc6cbe-beba-480c-8a90-9da0a622e663 is now active with session: 06be693d-7e07-47d0-a3bb-b792ebb38fec","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176803583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59132024-02-13T21:41:40.399Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:39.176836502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354,"session_id":"06be693d-7e07-47d0-a3bb-b792ebb38fec"}
59142024-02-13T21:41:40.399Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59152024-02-13T21:41:40.399Z2024-02-13T21:41:39.176865531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
59162024-02-13T21:41:40.399ZLoading write count information from file "/tmp/test_repair_verify.data"
59172024-02-13T21:41:40.399ZRun Repair workload
59182024-02-13T21:41:40.399Z01/30 Write block 455 len 4096 data: 4
59192024-02-13T21:41:40.399Z02/30 Read block 331 len 24576
59202024-02-13T21:41:40.399Z03/30 Write block 312 len 40960 data: 7 5 5 5 5 5 4 5 5 5
59212024-02-13T21:41:40.400Z04/30 Write block 98 len 20480 data: 5 5 5 5 9
59222024-02-13T21:41:40.400Z05/30 Write block 519 len 8192 data: 15 16
59232024-02-13T21:41:40.400Z06/30 Read block 279 len 4096
59242024-02-13T21:41:40.400Z07/30 Write block 563 len 20480 data: 2 5 6 6 6
59252024-02-13T21:41:40.400Z08/30 Write block 329 len 36864 data: 2 2 5 7 7 7 8 8 7
59262024-02-13T21:41:40.400Z09/30 Write block 204 len 8192 data: 5 5
59272024-02-13T21:41:40.400Z10/30 Read block 450 len 16384
59282024-02-13T21:41:40.400Z11/30 Read block 532 len 28672
59292024-02-13T21:41:40.400Z12/30 Write block 136 len 24576 data: 5 4 5 3 4 5
59302024-02-13T21:41:40.400Z13/30 Flush
59312024-02-13T21:41:40.400Z14/30 Write block 464 len 28672 data: 5 5 4 5 4 6 6
59322024-02-13T21:41:40.400Z15/30 Read block 76 len 4096
59332024-02-13T21:41:40.404Z16/30 Write block 572 len 16384 data: 4 3 4 4
59342024-02-13T21:41:40.404Z17/30 Write block 68 len 40960 data: 5 5 6 6 7 5 6 6 6 6
59352024-02-13T21:41:40.404Z18/30 Write block 46 len 4096 data: 3
59362024-02-13T21:41:40.404Z19/30 Flush
59372024-02-13T21:41:40.404Z20/30 Write block 275 len 24576 data: 5 5 6 6 6 5
59382024-02-13T21:41:40.404Z21/30 Read block 515 len 40960
59392024-02-13T21:41:40.404Z22/30 Read block 253 len 4096
59402024-02-13T21:41:40.404Z23/30 Flush
59412024-02-13T21:41:40.404Z24/30 Write block 514 len 16384 data: 10 9 8 10
59422024-02-13T21:41:40.404Z25/30 Write block 114 len 24576 data: 3 3 5 5 5 5
59432024-02-13T21:41:40.404Z26/30 Read block 502 len 16384
59442024-02-13T21:41:40.408Z27/30 Read block 466 len 8192
59452024-02-13T21:41:40.408Z28/30 Read block 129 len 4096
59462024-02-13T21:41:40.408Z29/30 Read block 78 len 16384
59472024-02-13T21:41:40.408Z30/30 Read block 44 len 24576
59482024-02-13T21:41:40.408Z----------------------------------------------------------------
59492024-02-13T21:41:40.408Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59502024-02-13T21:41:40.408ZStates: Active Active Active
59512024-02-13T21:41:40.408ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59522024-02-13T21:41:40.408Z 1 Acked 1000 Write 1 Done Done Done false
59532024-02-13T21:41:40.408Z 2 Acked 1001 Read 6 Done Done Done false
59542024-02-13T21:41:40.408Z 3 Acked 1002 Write 10 Done Done Done false
59552024-02-13T21:41:40.408Z 4 Acked 1003 Write 5 Done Done Done false
59562024-02-13T21:41:40.408Z 5 Acked 1004 Write 2 Done Done Done false
59572024-02-13T21:41:40.409Z 6 Acked 1005 Read 1 Done Sent Done false
59582024-02-13T21:41:40.409Z 7 Acked 1006 Write 5 Done Sent Done false
59592024-02-13T21:41:40.409Z 8 Acked 1007 Write 9 Done Sent Done false
59602024-02-13T21:41:40.409Z 9 Acked 1008 Write 2 Done Sent Done false
59612024-02-13T21:41:40.409Z 10 Acked 1009 Read 4 Done Sent Done false
59622024-02-13T21:41:40.409Z 11 Acked 1010 Read 7 Done Sent Done false
59632024-02-13T21:41:40.409Z 12 Acked 1011 Write 6 Done Sent Done false
59642024-02-13T21:41:40.409Z 13 Acked 1012 Flush 0 Done Sent Done false
59652024-02-13T21:41:40.409Z 14 Acked 1013 Write 7 Done Sent Done false
59662024-02-13T21:41:40.409Z 15 Acked 1014 Read 1 Done Sent Done false
59672024-02-13T21:41:40.409Z 16 Acked 1015 Write 4 Done Sent Done false
59682024-02-13T21:41:40.409Z 17 Acked 1016 Write 10 Done Sent Done false
59692024-02-13T21:41:40.409Z 18 Acked 1017 Write 1 Done Sent Done false
59702024-02-13T21:41:40.409Z 19 Acked 1018 Flush 0 Done Sent Done false
59712024-02-13T21:41:40.409Z 20 Acked 1019 Write 6 Done Sent Done false
59722024-02-13T21:41:40.409Z 21 Acked 1020 Read 10 Done Sent Done false
59732024-02-13T21:41:40.409Z 22 Acked 1021 Read 1 Done Sent Done false
59742024-02-13T21:41:40.409Z 23 Acked 1022 Flush 0 Done Sent Done false
59752024-02-13T21:41:40.409Z 24 Acked 1023 Write 4 Done Sent Done false
59762024-02-13T21:41:40.409Z 25 Acked 1024 Write 6 Done Sent Done false
59772024-02-13T21:41:40.409Z 26 Acked 1025 Read 4 Done Sent Done false
59782024-02-13T21:41:40.409Z 27 Acked 1026 Read 2 Done Sent Done false
59792024-02-13T21:41:40.410Z 28 Acked 1027 Read 1 Done Sent Done false
59802024-02-13T21:41:40.410Z 29 Acked 1028 Read 4 Done Sent Done false
59812024-02-13T21:41:40.410Z 30 Acked 1029 Read 6 Done Sent Done false
59822024-02-13T21:41:40.410Z STATES DS:0 DS:1 DS:2 TOTAL
59832024-02-13T21:41:40.410Z New 0 0 0 0
59842024-02-13T21:41:40.410Z Sent 0 25 0 25
59852024-02-13T21:41:40.410Z Done 30 5 30 65
59862024-02-13T21:41:40.410Z Skipped 0 0 0 0
59872024-02-13T21:41:40.410Z Error 0 0 0 0
59882024-02-13T21:41:40.410ZLast Flush: 1022 0 1022
59892024-02-13T21:41:40.410ZDownstairs last five completed:
59902024-02-13T21:41:40.410ZUpstairs last five completed: 30 29 28 27 26
59912024-02-13T21:41:40.410Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:39.208560914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1354}
59922024-02-13T21:41:40.414ZWrote out file "/tmp/test_repair_verify.data"
59932024-02-13T21:41:40.414Z
59942024-02-13T21:41:40.418ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59952024-02-13T21:41:40.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59962024-02-13T21:41:40.431Z 0 000-019 52 52 52 83 83 83 F F F
59972024-02-13T21:41:40.431Z 1 020-039 51 51 51 82 82 82 F F F
59982024-02-13T21:41:40.431Z 2 040-059 54 48 54 86 76 86 F F F <---
59992024-02-13T21:41:40.434Z 3 060-079 54 52 54 86 83 86 F F F <---
60002024-02-13T21:41:40.434Z 4 080-099 54 50 54 85 81 85 F T F <---
60012024-02-13T21:41:40.434Z 5 100-119 54 50 54 85 81 85 T T T <---
60022024-02-13T21:41:40.434Z 6 120-139 54 53 54 85 84 85 F F F <---
60032024-02-13T21:41:40.434Z 7 140-159 54 48 54 85 76 85 F F F <---
60042024-02-13T21:41:40.434Z 8 160-179 53 53 53 84 84 84 F F F
60052024-02-13T21:41:40.434Z 9 180-199 48 48 48 76 76 76 F F F
60062024-02-13T21:41:40.434Z 10 200-219 54 44 54 85 67 85 F F F <---
60072024-02-13T21:41:40.434Z 11 220-239 50 50 50 79 79 79 F F F
60082024-02-13T21:41:40.434Z 12 240-259 51 51 51 82 82 82 F F F
60092024-02-13T21:41:40.434Z 13 260-279 54 47 54 87 75 87 F F F <---
60102024-02-13T21:41:40.435Z 14 280-299 54 47 54 87 75 87 F F F <---
60112024-02-13T21:41:40.435Z 15 300-319 54 48 54 85 76 85 F T F <---
60122024-02-13T21:41:40.435Z 16 320-339 54 53 54 85 84 85 F T F <---
60132024-02-13T21:41:40.435Z 17 340-359 53 53 53 84 84 84 F F F
60142024-02-13T21:41:40.435Z 18 360-379 53 53 53 84 84 84 F F F
60152024-02-13T21:41:40.435Z 19 380-399 45 45 45 71 71 71 F F F
60162024-02-13T21:41:40.435Z 20 400-419 53 53 53 84 84 84 F F F
60172024-02-13T21:41:40.435Z 21 420-439 50 50 50 80 80 80 F F F
60182024-02-13T21:41:40.435Z 22 440-459 54 47 54 85 75 85 F T F <---
60192024-02-13T21:41:40.435Z 23 460-479 54 50 54 86 77 86 F F F <---
60202024-02-13T21:41:40.435Z 24 480-499 50 50 50 81 81 81 F F F
60212024-02-13T21:41:40.435Z 25 500-519 54 53 54 85 84 85 T T T <---
60222024-02-13T21:41:40.435Z 26 520-539 54 51 54 85 82 85 F T F <---
60232024-02-13T21:41:40.435Z 27 540-559 48 48 48 76 76 76 F F F
60242024-02-13T21:41:40.435Z 28 560-579 54 44 54 86 67 86 F F F <---
60252024-02-13T21:41:40.435Z 29 580-599 53 53 53 84 84 84 F F F
60262024-02-13T21:41:40.435ZMax gen: 54, Max flush: 87
60272024-02-13T21:41:40.435ZError: Difference in extent metadata found!
60282024-02-13T21:41:40.435ZA Difference in extent metadata is expected here
60292024-02-13T21:41:40.435ZOn loop 27
60302024-02-13T21:41:40.435Z
60312024-02-13T21:41:40.439ZVerifying data now
60322024-02-13T21:41:46.213ZNo change after verify
60332024-02-13T21:41:46.218ZLoop: 27 Downstairs dump after verify (and repair):
60342024-02-13T21:41:46.218ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60352024-02-13T21:41:46.218Z 0 000-019 52 52 52 83 83 83 F F F
60362024-02-13T21:41:46.218Z 1 020-039 51 51 51 82 82 82 F F F
60372024-02-13T21:41:46.219Z 2 040-059 54 54 54 86 86 86 F F F
60382024-02-13T21:41:46.219Z 3 060-079 54 54 54 86 86 86 F F F
60392024-02-13T21:41:46.219Z 4 080-099 54 54 54 85 85 85 F F F
60402024-02-13T21:41:46.219Z 5 100-119 55 55 55 88 88 88 F F F
60412024-02-13T21:41:46.219Z 6 120-139 54 54 54 85 85 85 F F F
60422024-02-13T21:41:46.219Z 7 140-159 54 54 54 85 85 85 F F F
60432024-02-13T21:41:46.219Z 8 160-179 53 53 53 84 84 84 F F F
60442024-02-13T21:41:46.220Z 9 180-199 48 48 48 76 76 76 F F F
60452024-02-13T21:41:46.220Z 10 200-219 54 54 54 85 85 85 F F F
60462024-02-13T21:41:46.220Z 11 220-239 50 50 50 79 79 79 F F F
60472024-02-13T21:41:46.220Z 12 240-259 51 51 51 82 82 82 F F F
60482024-02-13T21:41:46.220Z 13 260-279 54 54 54 87 87 87 F F F
60492024-02-13T21:41:46.220Z 14 280-299 54 54 54 87 87 87 F F F
60502024-02-13T21:41:46.220Z 15 300-319 54 54 54 85 85 85 F F F
60512024-02-13T21:41:46.220Z 16 320-339 54 54 54 85 85 85 F F F
60522024-02-13T21:41:46.221Z 17 340-359 53 53 53 84 84 84 F F F
60532024-02-13T21:41:46.221Z 18 360-379 53 53 53 84 84 84 F F F
60542024-02-13T21:41:46.221Z 19 380-399 45 45 45 71 71 71 F F F
60552024-02-13T21:41:46.221Z 20 400-419 53 53 53 84 84 84 F F F
60562024-02-13T21:41:46.221Z 21 420-439 50 50 50 80 80 80 F F F
60572024-02-13T21:41:46.221Z 22 440-459 54 54 54 85 85 85 F F F
60582024-02-13T21:41:46.222Z 23 460-479 54 54 54 86 86 86 F F F
60592024-02-13T21:41:46.222Z 24 480-499 50 50 50 81 81 81 F F F
60602024-02-13T21:41:46.222Z 25 500-519 55 55 55 88 88 88 F F F
60612024-02-13T21:41:46.222Z 26 520-539 54 54 54 85 85 85 F F F
60622024-02-13T21:41:46.222Z 27 540-559 48 48 48 76 76 76 F F F
60632024-02-13T21:41:46.222Z 28 560-579 54 54 54 86 86 86 F F F
60642024-02-13T21:41:46.223Z 29 580-599 53 53 53 84 84 84 F F F
60652024-02-13T21:41:46.223ZMax gen: 55, Max flush: 88
60662024-02-13T21:41:46.223Z
60672024-02-13T21:41:46.223ZBegin loop 28
60682024-02-13T21:41:46.223ZDownstairs to restart: 2
60692024-02-13T21:41:46.232Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030232783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
60702024-02-13T21:41:46.232Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030466668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
60712024-02-13T21:41:46.244Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030501985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
60722024-02-13T21:41:46.244Z{"msg":"Crucible cba128ee-f01a-45d3-a000-aac692e891f9 has session id: 6f92c616-6bff-4351-a7a4-649cf1701004","v":0,"name":"crucible","level":30Crucible runtime is spawned
60732024-02-13T21:41:46.244Z,"time":"2024-02-13T21:41:45.030562583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60742024-02-13T21:41:46.245Z{"msg":"Upstairs opts: Upstairs UUID: cba128ee-f01a-45d3-a000-aac692e891f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030600969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60752024-02-13T21:41:46.245Z{"msg":"Crucible stats registered with UUID: cba128ee-f01a-45d3-a000-aac692e891f9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030624151Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60762024-02-13T21:41:46.245Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030659838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
60772024-02-13T21:41:46.245Z{"msg":"cba128ee-f01a-45d3-a000-aac692e891f9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030726594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60782024-02-13T21:41:46.245Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030780834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60792024-02-13T21:41:46.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030815781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60802024-02-13T21:41:46.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030864534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60812024-02-13T21:41:46.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030911457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60822024-02-13T21:41:46.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.030950083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60832024-02-13T21:41:46.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031000664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60842024-02-13T21:41:46.246Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031033133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60852024-02-13T21:41:46.247Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031136535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60862024-02-13T21:41:46.247Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031174071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60872024-02-13T21:41:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031200302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60882024-02-13T21:41:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031224213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60892024-02-13T21:41:46.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031267208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"io task","client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60902024-02-13T21:41:46.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031879946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60912024-02-13T21:41:46.247Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031927528Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60922024-02-13T21:41:46.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031957567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60932024-02-13T21:41:46.247Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.031992985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60942024-02-13T21:41:46.247Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.032525451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60952024-02-13T21:41:46.248Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.032664371Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60962024-02-13T21:41:46.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.032955505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60972024-02-13T21:41:46.248Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.03299611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60982024-02-13T21:41:46.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033023581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
60992024-02-13T21:41:46.248Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033133761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61002024-02-13T21:41:46.248Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033186811Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61012024-02-13T21:41:46.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033309677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61022024-02-13T21:41:46.248Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033341016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61032024-02-13T21:41:46.249Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033365077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61042024-02-13T21:41:46.249Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.033652283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61052024-02-13T21:41:46.249Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034066822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61062024-02-13T21:41:46.249Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034102729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61072024-02-13T21:41:46.249Z{"msg":"[0]R flush_numbers[0..12]: [83, 82, 86, 86, 85, 88, 85, 85, 84, 76, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034119103Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61082024-02-13T21:41:46.249Z{"msg":"[0]R generation[0..12]: [52, 51, 54, 54, 54, 55, 54, 54, 53, 48, 54, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034140835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61092024-02-13T21:41:46.249Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.03415573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61102024-02-13T21:41:46.249Z{"msg":"[1]R flush_numbers[0..12]: [83, 82, 86, 86, 85, 88, 85, 85, 84, 76, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034176642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61112024-02-13T21:41:46.249Z{"msg":"[1]R generation[0..12]: [52, 51, 54, 54, 54, 55, 54, 54, 53, 48, 54, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034190867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61122024-02-13T21:41:46.250Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034209221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61132024-02-13T21:41:46.250Z{"msg":"[2]R flush_numbers[0..12]: [83, 82, 86, 86, 85, 88, 85, 85, 84, 76, 85, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034223096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61142024-02-13T21:41:46.250Z{"msg":"[2]R generation[0..12]: [52, 51, 54, 54, 54, 55, 54, 54, 53, 48, 54, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034248966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61152024-02-13T21:41:46.250Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
61162024-02-13T21:41:46.250Z,"time":"2024-02-13T21:41:45.034263331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61172024-02-13T21:41:46.250Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034277616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61182024-02-13T21:41:46.250Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034304286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61192024-02-13T21:41:46.250Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034318671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61202024-02-13T21:41:46.250Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034331936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"":"downstairs","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61212024-02-13T21:41:46.251Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034345411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61222024-02-13T21:41:46.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034357677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"0","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61232024-02-13T21:41:46.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034374401Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"1","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61242024-02-13T21:41:46.251Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034398212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"client":"2","":"downstairs","session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61252024-02-13T21:41:46.251Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034432999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61262024-02-13T21:41:46.251Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034455451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61272024-02-13T21:41:46.251Z{"msg":"cba128ee-f01a-45d3-a000-aac692e891f9 is now active with session: 6f92c616-6bff-4351-a7a4-649cf1701004","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034489739Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61282024-02-13T21:41:46.251Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.03451428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362,"session_id":"6f92c616-6bff-4351-a7a4-649cf1701004"}
61292024-02-13T21:41:46.251Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:45.034528225Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
61302024-02-13T21:41:46.251ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61312024-02-13T21:41:46.252ZLoading write count information from file "/tmp/test_repair_verify.data"
61322024-02-13T21:41:46.252ZRun Repair workload
61332024-02-13T21:41:46.252Z01/30 Write block 214 len 24576 data: 5 5 7 8 7 7
61342024-02-13T21:41:46.252Z02/30 Flush
61352024-02-13T21:41:46.256Z03/30 Write block 335 len 36864 data: 9 9 8 7 6 6 5 5 5
61362024-02-13T21:41:46.256Z04/30 Read block 118 len 16384
61372024-02-13T21:41:46.257Z05/30 Flush
61382024-02-13T21:41:46.257Z06/30 Write block 128 len 32768 data: 5 5 3 7 7 7 5 6
61392024-02-13T21:41:46.257Z07/30 Flush
61402024-02-13T21:41:46.257Z08/30 Write block 290 len 20480 data: 5 5 5 6 6
61412024-02-13T21:41:46.257Z09/30 Read block 564 len 40960
61422024-02-13T21:41:46.257Z10/30 Read block 65 len 40960
61432024-02-13T21:41:46.257Z11/30 Write block 292 len 28672 data: 6 7 7 6 6 6 5
61442024-02-13T21:41:46.258Z12/30 Write block 183 len 28672 data: 6 6 8 7 5 7 7
61452024-02-13T21:41:46.258Z13/30 Write block 596 len 4096 data: 4
61462024-02-13T21:41:46.258Z14/30 Read block 332 len 24576
61472024-02-13T21:41:46.258Z15/30 Read block 157 len 16384
61482024-02-13T21:41:46.262Z16/30 Write block 459 len 36864 data: 7 6 5 5 4 6 6 5 6
61492024-02-13T21:41:46.262Z17/30 Write block 72 len 16384 data: 8 6 7 7
61502024-02-13T21:41:46.262Z18/30 Write block 310 len 16384 data: 4 6 8 6
61512024-02-13T21:41:46.262Z19/30 Read block 385 len 4096
61522024-02-13T21:41:46.262Z20/30 Read block 199 len 28672
61532024-02-13T21:41:46.262Z21/30 Read block 317 len 40960
61542024-02-13T21:41:46.262Z22/30 Read block 555 len 40960
61552024-02-13T21:41:46.267Z23/30 Read block 189 len 24576
61562024-02-13T21:41:46.267Z24/30 Write block 125 len 4096 data: 7
61572024-02-13T21:41:46.267Z25/30 Read block 152 len 28672
61582024-02-13T21:41:46.268Z26/30 Write block 429 len 16384 data: 7 7 8 8
61592024-02-13T21:41:46.268Z27/30 Write block 437 len 24576 data: 3 4 4 4 5 5
61602024-02-13T21:41:46.268Z28/30 Read block 197 len 4096
61612024-02-13T21:41:46.268Z29/30 Write block 323 len 16384 data: 6 5 6 6
61622024-02-13T21:41:46.273Z30/30 Write block 271 len 40960 data: 4 4 4 4 6 6 7 7 7 6
61632024-02-13T21:41:46.273Z----------------------------------------------------------------
61642024-02-13T21:41:46.273Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:28
61652024-02-13T21:41:46.273ZStates: Active Active Active
61662024-02-13T21:41:46.273ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61672024-02-13T21:41:46.273Z 3 Acked 1002 Write 9 Done Done Done false
61682024-02-13T21:41:46.274Z 4 Acked 1003 Read 4 Done Done Done false
61692024-02-13T21:41:46.274Z 5 Acked 1004 Flush 0 Done Done Sent false
61702024-02-13T21:41:46.274Z 6 Acked 1005 Write 8 Done Done Sent false
61712024-02-13T21:41:46.274Z 7 Acked 1006 Flush 0 Done Done Sent false
61722024-02-13T21:41:46.274Z 8 Acked 1007 Write 5 Done Done Sent false
61732024-02-13T21:41:46.275Z 9 Acked 1008 Read 10 Done Done Sent false
61742024-02-13T21:41:46.275Z 10 Acked 1009 Read 10 Done Done Sent false
61752024-02-13T21:41:46.275Z 11 Acked 1010 Write 7 Done Done Sent false
61762024-02-13T21:41:46.275Z 12 Acked 1011 Write 7 Done Done Sent false
61772024-02-13T21:41:46.275Z 13 Acked 1012 Write 1 Done Done Sent false
61782024-02-13T21:41:46.275Z 14 Acked 1013 Read 6 Done Done Sent false
61792024-02-13T21:41:46.275Z 15 Acked 1014 Read 4 Done Done Sent false
61802024-02-13T21:41:46.275Z 16 Acked 1015 Write 9 Done Done Sent false
61812024-02-13T21:41:46.276Z 17 Acked 1016 Write 4 Done Done Sent false
61822024-02-13T21:41:46.276Z 18 Acked 1017 Write 4 Done Done Sent false
61832024-02-13T21:41:46.276Z 19 Acked 1018 Read 1 Done Done Sent false
61842024-02-13T21:41:46.276Z 20 Acked 1019 Read 7 Done Done Sent false
61852024-02-13T21:41:46.276Z 21 Acked 1020 Read 10 Done Done Sent false
61862024-02-13T21:41:46.276Z 22 Acked 1021 Read 10 Done Done Sent false
61872024-02-13T21:41:46.276Z 23 Acked 1022 Read 6 Done Done Sent false
61882024-02-13T21:41:46.277Z 24 Acked 1023 Write 1 Done Done Sent false
61892024-02-13T21:41:46.277Z 25 Acked 1024 Read 7 Done Done Sent false
61902024-02-13T21:41:46.277Z 26 Acked 1025 Write 4 Done Done Sent false
61912024-02-13T21:41:46.277Z 27 Acked 1026 Write 6 Done Done Sent false
61922024-02-13T21:41:46.277Z 28 Acked 1027 Read 1 Done Done Sent false
61932024-02-13T21:41:46.277Z 29 Acked 1028 Write 4 Done Done Sent false
61942024-02-13T21:41:46.277Z 30 Acked 1029 Write 10 Sent Sent Sent false
61952024-02-13T21:41:46.277Z STATES DS:0 DS:1 DS:2 TOTAL
61962024-02-13T21:41:46.278Z New 0 0 0 0
61972024-02-13T21:41:46.278Z Sent 1 1 26 28
61982024-02-13T21:41:46.278Z Done 27 27 2 56
61992024-02-13T21:41:46.278Z Skipped 0 0 0 0
62002024-02-13T21:41:46.278Z Error 0 0 0 0
62012024-02-13T21:41:46.278ZLast Flush: 1006 1006 1001
62022024-02-13T21:41:46.279ZDownstairs last five completed: 1001 1000
62032024-02-13T21:41:46.279ZUpstairs last five completed: 30 29 28 27 26
62042024-02-13T21:41:46.279Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:45.066484623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1362}
62052024-02-13T21:41:46.279ZWrote out file "/tmp/test_repair_verify.data"
62062024-02-13T21:41:46.279Z
62072024-02-13T21:41:46.284ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62082024-02-13T21:41:46.295ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62092024-02-13T21:41:46.295Z 0 000-019 52 52 52 83 83 83 F F F
62102024-02-13T21:41:46.295Z 1 020-039 51 51 51 82 82 82 F F F
62112024-02-13T21:41:46.295Z 2 040-059 54 54 54 86 86 86 F F F
62122024-02-13T21:41:46.295Z 3 060-079 54 54 54 86 86 86 T T F <---
62132024-02-13T21:41:46.299Z 4 080-099 54 54 54 85 85 85 F F F
62142024-02-13T21:41:46.300Z 5 100-119 55 55 55 88 88 88 F F F
62152024-02-13T21:41:46.300Z 6 120-139 56 56 54 91 91 85 T T F <---
62162024-02-13T21:41:46.300Z 7 140-159 54 54 54 85 85 85 F F F
62172024-02-13T21:41:46.300Z 8 160-179 53 53 53 84 84 84 F F F
62182024-02-13T21:41:46.300Z 9 180-199 48 48 48 76 76 76 T T F <---
62192024-02-13T21:41:46.300Z 10 200-219 56 56 56 89 89 89 F F F
62202024-02-13T21:41:46.300Z 11 220-239 50 50 50 79 79 79 F F F
62212024-02-13T21:41:46.300Z 12 240-259 51 51 51 82 82 82 F F F
62222024-02-13T21:41:46.300Z 13 260-279 54 54 54 87 87 87 T T F <---
62232024-02-13T21:41:46.301Z 14 280-299 54 54 54 87 87 87 T T F <---
62242024-02-13T21:41:46.301Z 15 300-319 54 54 54 85 85 85 T T F <---
62252024-02-13T21:41:46.301Z 16 320-339 56 56 54 90 90 85 T T T <---
62262024-02-13T21:41:46.301Z 17 340-359 56 56 53 90 90 84 F F T <---
62272024-02-13T21:41:46.301Z 18 360-379 53 53 53 84 84 84 F F F
62282024-02-13T21:41:46.301Z 19 380-399 45 45 45 71 71 71 F F F
62292024-02-13T21:41:46.301Z 20 400-419 53 53 53 84 84 84 F F F
62302024-02-13T21:41:46.302Z 21 420-439 50 50 50 80 80 80 T T F <---
62312024-02-13T21:41:46.302Z 22 440-459 54 54 54 85 85 85 T T F <---
62322024-02-13T21:41:46.302Z 23 460-479 54 54 54 86 86 86 T T F <---
62332024-02-13T21:41:46.302Z 24 480-499 50 50 50 81 81 81 F F F
62342024-02-13T21:41:46.302Z 25 500-519 55 55 55 88 88 88 F F F
62352024-02-13T21:41:46.302Z 26 520-539 54 54 54 85 85 85 F F F
62362024-02-13T21:41:46.302Z 27 540-559 48 48 48 76 76 76 F F F
62372024-02-13T21:41:46.302Z 28 560-579 54 54 54 86 86 86 F F F
62382024-02-13T21:41:46.302Z 29 580-599 53 53 53 84 84 84 T T F <---
62392024-02-13T21:41:46.302ZMax gen: 56, Max flush: 91
62402024-02-13T21:41:46.302ZError: Difference in extent metadata found!
62412024-02-13T21:41:46.303ZA Difference in extent metadata is expected here
62422024-02-13T21:41:46.303ZOn loop 28
62432024-02-13T21:41:46.303Z
62442024-02-13T21:41:46.309ZVerifying data now
62452024-02-13T21:41:50.930ZNo change after verify
62462024-02-13T21:41:50.930ZLoop: 28 Downstairs dump after verify (and repair):
62472024-02-13T21:41:50.948ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62482024-02-13T21:41:50.948Z 0 000-019 52 52 52 83 83 83 F F F
62492024-02-13T21:41:50.948Z 1 020-039 51 51 51 82 82 82 F F F
62502024-02-13T21:41:50.948Z 2 040-059 54 54 54 86 86 86 F F F
62512024-02-13T21:41:50.948Z 3 060-079 57 57 57 92 92 92 F F F
62522024-02-13T21:41:50.949Z 4 080-099 54 54 54 85 85 85 F F F
62532024-02-13T21:41:50.949Z 5 100-119 55 55 55 88 88 88 F F F
62542024-02-13T21:41:50.949Z 6 120-139 57 57 57 92 92 92 F F F
62552024-02-13T21:41:50.949Z 7 140-159 54 54 54 85 85 85 F F F
62562024-02-13T21:41:50.949Z 8 160-179 53 53 53 84 84 84 F F F
62572024-02-13T21:41:50.949Z 9 180-199 57 57 57 92 92 92 F F F
62582024-02-13T21:41:50.949Z 10 200-219 56 56 56 89 89 89 F F F
62592024-02-13T21:41:50.949Z 11 220-239 50 50 50 79 79 79 F F F
62602024-02-13T21:41:50.949Z 12 240-259 51 51 51 82 82 82 F F F
62612024-02-13T21:41:50.949Z 13 260-279 57 57 57 92 92 92 F F F
62622024-02-13T21:41:50.949Z 14 280-299 57 57 57 92 92 92 F F F
62632024-02-13T21:41:50.949Z 15 300-319 57 57 57 92 92 92 F F F
62642024-02-13T21:41:50.950Z 16 320-339 57 57 57 92 92 92 F F F
62652024-02-13T21:41:50.950Z 17 340-359 56 56 56 90 90 90 F F F
62662024-02-13T21:41:50.950Z 18 360-379 53 53 53 84 84 84 F F F
62672024-02-13T21:41:50.950Z 19 380-399 45 45 45 71 71 71 F F F
62682024-02-13T21:41:50.950Z 20 400-419 53 53 53 84 84 84 F F F
62692024-02-13T21:41:50.950Z 21 420-439 57 57 57 92 92 92 F F F
62702024-02-13T21:41:50.950Z 22 440-459 57 57 57 92 92 92 F F F
62712024-02-13T21:41:50.950Z 23 460-479 57 57 57 92 92 92 F F F
62722024-02-13T21:41:50.950Z 24 480-499 50 50 50 81 81 81 F F F
62732024-02-13T21:41:50.950Z 25 500-519 55 55 55 88 88 88 F F F
62742024-02-13T21:41:50.950Z 26 520-539 54 54 54 85 85 85 F F F
62752024-02-13T21:41:50.951Z 27 540-559 48 48 48 76 76 76 F F F
62762024-02-13T21:41:50.951Z 28 560-579 54 54 54 86 86 86 F F F
62772024-02-13T21:41:50.951Z 29 580-599 57 57 57 92 92 92 F F F
62782024-02-13T21:41:50.951ZMax gen: 57, Max flush: 92
62792024-02-13T21:41:50.973Z
62802024-02-13T21:41:50.973ZBegin loop 29
62812024-02-13T21:41:50.973ZDownstairs to restart: 1
62822024-02-13T21:41:50.973Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755129372Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
62832024-02-13T21:41:50.974Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.75538162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
62842024-02-13T21:41:50.974Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755401353Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
62852024-02-13T21:41:50.974Z{"msg":"Crucible ecdc02fa-617a-450c-8c9d-929a317ec3aa has session id: 6d74cc4d-ee1d-482e-a0ab-c101b3418d33","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755432642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62862024-02-13T21:41:50.974ZCrucible runtime is spawned
62872024-02-13T21:41:50.974Z{"msg":"Upstairs opts: Upstairs UUID: ecdc02fa-617a-450c-8c9d-929a317ec3aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755486902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62882024-02-13T21:41:50.974Z{"msg":"Crucible stats registered with UUID: ecdc02fa-617a-450c-8c9d-929a317ec3aa","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755517491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62892024-02-13T21:41:50.974Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755540053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
62902024-02-13T21:41:50.974Z{"msg":"ecdc02fa-617a-450c-8c9d-929a317ec3aa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755677033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62912024-02-13T21:41:50.974Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.75571221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62922024-02-13T21:41:50.974Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755730763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62932024-02-13T21:41:50.975Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755763462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62942024-02-13T21:41:50.975Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755787993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62952024-02-13T21:41:50.975Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755811974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62962024-02-13T21:41:50.975Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755836345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62972024-02-13T21:41:50.975Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755930211Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62982024-02-13T21:41:50.975Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.755973815Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
62992024-02-13T21:41:50.975Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756001905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63002024-02-13T21:41:50.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756039502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63012024-02-13T21:41:50.975Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756184469Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63022024-02-13T21:41:50.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756232581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"io task","client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63032024-02-13T21:41:50.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756686447Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63042024-02-13T21:41:50.985Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756740297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63052024-02-13T21:41:50.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756806803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63062024-02-13T21:41:50.985Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.756836053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63072024-02-13T21:41:50.986Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757305102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63082024-02-13T21:41:50.986Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757430127Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63092024-02-13T21:41:50.986Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757726659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63102024-02-13T21:41:50.986Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757777491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63112024-02-13T21:41:50.986Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757854663Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63122024-02-13T21:41:50.987Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757896268Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63132024-02-13T21:41:50.987Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757932795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63142024-02-13T21:41:50.987Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757954177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63152024-02-13T21:41:50.987Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757981087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63162024-02-13T21:41:50.987Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.757996581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63172024-02-13T21:41:50.987Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758214213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63182024-02-13T21:41:50.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758620075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63192024-02-13T21:41:50.988Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758654213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63202024-02-13T21:41:50.988Z{"msg":"[0]R flush_numbers[0..12]: [83, 82, 86, 92, 85, 88, 92, 85, 84, 92, 89, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758719669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63212024-02-13T21:41:50.988Z{"msg":"[0]R generation[0..12]: [52, 51, 54, 57, 54, 55, 57, 54, 53, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758756336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63222024-02-13T21:41:50.988Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.75877326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63232024-02-13T21:41:50.988Z{"msg":"[1]R flush_numbers[0..12]: [83, 82, 86, 92, 85, 88, 92, 85, 84, 92, 89, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758794622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63242024-02-13T21:41:50.988Z{"msg":"[1]R generation[0..12]: [52, 51, 54, 57, 54, 55, 57, 54, 53, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758808747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63252024-02-13T21:41:50.988Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758823641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63262024-02-13T21:41:50.988Z{"msg":"[2]R flush_numbers[0..12]: [83, 82, 86, 92, 85, 88, 92, 85, 84, 92, 89, 79]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
63272024-02-13T21:41:50.989Z,"time":"2024-02-13T21:41:49.758844684Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63282024-02-13T21:41:50.989Z{"msg":"[2]R generation[0..12]: [52, 51, 54, 57, 54, 55, 57, 54, 53, 57, 56, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758859099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63292024-02-13T21:41:50.989Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758879751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63302024-02-13T21:41:50.989Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758894866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63312024-02-13T21:41:50.989Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758915368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63322024-02-13T21:41:50.989Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758929953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63332024-02-13T21:41:50.989Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758963251Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"":"downstairs","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63342024-02-13T21:41:50.990Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.758983803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63352024-02-13T21:41:50.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759003776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"0","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63362024-02-13T21:41:50.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759017541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"1","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63372024-02-13T21:41:50.990Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759030586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"client":"2","":"downstairs","session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63382024-02-13T21:41:50.990Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759044401Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63392024-02-13T21:41:50.990Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759057097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63402024-02-13T21:41:50.990Z{"msg":"ecdc02fa-617a-450c-8c9d-929a317ec3aa is now active with session: 6d74cc4d-ee1d-482e-a0ab-c101b3418d33","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759077209Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63412024-02-13T21:41:50.990Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759089625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370,"session_id":"6d74cc4d-ee1d-482e-a0ab-c101b3418d33"}
63422024-02-13T21:41:50.991Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:49.759108188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
63432024-02-13T21:41:50.991ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63442024-02-13T21:41:50.991ZLoading write count information from file "/tmp/test_repair_verify.data"
63452024-02-13T21:41:50.991ZRun Repair workload
63462024-02-13T21:41:50.991Z01/30 Write block 299 len 24576 data: 4 4 3 3 3 3
63472024-02-13T21:41:50.991Z02/30 Read block 130 len 4096
63482024-02-13T21:41:50.991Z03/30 Read block 533 len 24576
63492024-02-13T21:41:50.992Z04/30 Write block 545 len 4096 data: 9
63502024-02-13T21:41:50.992Z05/30 Flush
63512024-02-13T21:41:50.992Z06/30 Write block 386 len 32768 data: 4 6 5 4 4 4 4 4
63522024-02-13T21:41:50.992Z07/30 Read block 111 len 20480
63532024-02-13T21:41:50.992Z08/30 Write block 263 len 32768 data: 4 6 6 6 6 6 4 4
63542024-02-13T21:41:50.992Z09/30 Flush
63552024-02-13T21:41:50.992Z10/30 Flush
63562024-02-13T21:41:50.993Z11/30 Write block 372 len 28672 data: 4 5 5 8 5 5 5
63572024-02-13T21:41:50.993Z12/30 Read block 85 len 28672
63582024-02-13T21:41:50.993Z13/30 Read block 548 len 16384
63592024-02-13T21:41:50.993Z14/30 Write block 209 len 8192 data: 4 5
63602024-02-13T21:41:50.993Z15/30 Read block 121 len 40960
63612024-02-13T21:41:50.993Z16/30 Read block 516 len 12288
63622024-02-13T21:41:51.000Z17/30 Write block 468 len 36864 data: 5 7 7 8 8 9 9 8 5
63632024-02-13T21:41:51.000Z18/30 Write block 435 len 20480 data: 6 4 4 5 5
63642024-02-13T21:41:51.000Z19/30 Read block 236 len 32768
63652024-02-13T21:41:51.000Z20/30 Flush
63662024-02-13T21:41:51.000Z21/30 Flush
63672024-02-13T21:41:51.001Z22/30 Write block 480 len 16384 data: 5 5 5 4
63682024-02-13T21:41:51.001Z23/30 Write block 69 len 8192 data: 6 7
63692024-02-13T21:41:51.001Z24/30 Flush
63702024-02-13T21:41:51.001Z25/30 Write block 218 len 4096 data: 8
63712024-02-13T21:41:51.001Z26/30 Flush
63722024-02-13T21:41:51.002Z27/30 Write block 101 len 24576 data: 6 10 9 8 7 5
63732024-02-13T21:41:51.002Z28/30 Read block 213 len 40960
63742024-02-13T21:41:51.002Z29/30 Write block 250 len 40960 data: 4 5 4 4 5 5 6 6 5 3
63752024-02-13T21:41:51.002Z30/30 Read block 166 len 12288
63762024-02-13T21:41:51.002Z----------------------------------------------------------------
63772024-02-13T21:41:51.002Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63782024-02-13T21:41:51.002ZStates: Active Active Active
63792024-02-13T21:41:51.003ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63802024-02-13T21:41:51.003Z 1 Acked 1000 Write 6 Done Done Done false
63812024-02-13T21:41:51.003Z 2 Acked 1001 Read 1 Done Done Done false
63822024-02-13T21:41:51.003Z 3 Acked 1002 Read 6 Done Sent Done false
63832024-02-13T21:41:51.003Z 4 Acked 1003 Write 1 Done Sent Done false
63842024-02-13T21:41:51.003Z 5 Acked 1004 Flush 0 Done Sent Done false
63852024-02-13T21:41:51.004Z 6 Acked 1005 Write 8 Done Sent Done false
63862024-02-13T21:41:51.004Z 7 Acked 1006 Read 5 Done Sent Done false
63872024-02-13T21:41:51.004Z 8 Acked 1007 Write 8 Done Sent Done false
63882024-02-13T21:41:51.004Z 9 Acked 1008 Flush 0 Done Sent Done false
63892024-02-13T21:41:51.004Z 10 Acked 1009 Flush 0 Done Sent Done false
63902024-02-13T21:41:51.004Z 11 Acked 1010 Write 7 Done Sent Done false
63912024-02-13T21:41:51.004Z 12 Acked 1011 Read 7 Done Sent Done false
63922024-02-13T21:41:51.004Z 13 Acked 1012 Read 4 Done Sent Done false
63932024-02-13T21:41:51.004Z 14 Acked 1013 Write 2 Done Sent Done false
63942024-02-13T21:41:51.004Z 15 Acked 1014 Read 10 Done Sent Done false
63952024-02-13T21:41:51.004Z 16 Acked 1015 Read 3 Done Sent Done false
63962024-02-13T21:41:51.004Z 17 Acked 1016 Write 9 Done Sent Done false
63972024-02-13T21:41:51.005Z 18 Acked 1017 Write 5 Done Sent Done false
63982024-02-13T21:41:51.005Z 19 Acked 1018 Read 8 Done Sent Done false
63992024-02-13T21:41:51.005Z 20 Acked 1019 Flush 0 Done Sent Done false
64002024-02-13T21:41:51.005Z 21 Acked 1020 Flush 0 Done Sent Done false
64012024-02-13T21:41:51.005Z 22 Acked 1021 Write 4 Done Sent Done false
64022024-02-13T21:41:51.005Z 23 Acked 1022 Write 2 Done Sent Done false
64032024-02-13T21:41:51.005Z 24 Acked 1023 Flush 0 Done Sent Done false
64042024-02-13T21:41:51.005Z 25 Acked 1024 Write 1 Done Sent Done false
64052024-02-13T21:41:51.005Z 26 Acked 1025 Flush 0 Done Sent Done false
64062024-02-13T21:41:51.005Z 27 Acked 1026 Write 6 Done Sent Done false
64072024-02-13T21:41:51.005Z 28 Acked 1027 Read 10 Done Sent Done false
64082024-02-13T21:41:51.005Z 29 Acked 1028 Write 10 Done Sent Done false
64092024-02-13T21:41:51.005Z 30 Acked 1029 Read 3 Done Sent Sent false
64102024-02-13T21:41:51.005Z STATES DS:0 DS:1 DS:2 TOTAL
64112024-02-13T21:41:51.005Z New 0 0 0 0
64122024-02-13T21:41:51.006Z Sent 0 28 1 29
64132024-02-13T21:41:51.006Z Done 30 2 29 61
64142024-02-13T21:41:51.006Z Skipped 0 0 0 0
64152024-02-13T21:41:51.006Z Error 0 0 0 0
64162024-02-13T21:41:51.006ZLast Flush: 1025 0 1025
64172024-02-13T21:41:51.006ZDownstairs last five completed:
64182024-02-13T21:41:51.006ZUpstairs last five completed: 30 29 28 27 26
64192024-02-13T21:41:51.006Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:49.793572397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1370}
64202024-02-13T21:41:51.006ZWrote out file "/tmp/test_repair_verify.data"
64212024-02-13T21:41:51.014Z
64222024-02-13T21:41:51.014ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64232024-02-13T21:41:51.020ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64242024-02-13T21:41:51.020Z 0 000-019 52 52 52 83 83 83 F F F
64252024-02-13T21:41:51.020Z 1 020-039 51 51 51 82 82 82 F F F
64262024-02-13T21:41:51.020Z 2 040-059 54 54 54 86 86 86 F F F
64272024-02-13T21:41:51.020Z 3 060-079 58 57 58 98 92 98 F F F <---
64282024-02-13T21:41:51.020Z 4 080-099 54 54 54 85 85 85 F F F
64292024-02-13T21:41:51.021Z 5 100-119 55 55 55 88 88 88 T F T <---
64302024-02-13T21:41:51.021Z 6 120-139 57 57 57 92 92 92 F F F
64312024-02-13T21:41:51.021Z 7 140-159 54 54 54 85 85 85 F F F
64322024-02-13T21:41:51.026Z 8 160-179 53 53 53 84 84 84 F F F
64332024-02-13T21:41:51.026Z 9 180-199 57 57 57 92 92 92 F F F
64342024-02-13T21:41:51.026Z 10 200-219 58 56 58 99 89 99 F F F <---
64352024-02-13T21:41:51.026Z 11 220-239 50 50 50 79 79 79 F F F
64362024-02-13T21:41:51.026Z 12 240-259 51 51 51 82 82 82 T F T <---
64372024-02-13T21:41:51.027Z 13 260-279 58 57 58 94 92 94 F F F <---
64382024-02-13T21:41:51.027Z 14 280-299 58 57 58 93 92 93 F T F <---
64392024-02-13T21:41:51.027Z 15 300-319 58 57 58 93 92 93 F T F <---
64402024-02-13T21:41:51.027Z 16 320-339 57 57 57 92 92 92 F F F
64412024-02-13T21:41:51.027Z 17 340-359 56 56 56 90 90 90 F F F
64422024-02-13T21:41:51.027Z 18 360-379 58 53 58 96 84 96 F F F <---
64432024-02-13T21:41:51.027Z 19 380-399 58 45 58 94 71 94 F F F <---
64442024-02-13T21:41:51.027Z 20 400-419 53 53 53 84 84 84 F F F
64452024-02-13T21:41:51.027Z 21 420-439 58 57 58 96 92 96 F F F <---
64462024-02-13T21:41:51.027Z 22 440-459 57 57 57 92 92 92 F F F
64472024-02-13T21:41:51.027Z 23 460-479 58 57 58 96 92 96 F F F <---
64482024-02-13T21:41:51.028Z 24 480-499 58 50 58 98 81 98 F F F <---
64492024-02-13T21:41:51.028Z 25 500-519 55 55 55 88 88 88 F F F
64502024-02-13T21:41:51.028Z 26 520-539 54 54 54 85 85 85 F F F
64512024-02-13T21:41:51.028Z 27 540-559 58 48 58 93 76 93 F F F <---
64522024-02-13T21:41:51.028Z 28 560-579 54 54 54 86 86 86 F F F
64532024-02-13T21:41:51.028Z 29 580-599 57 57 57 92 92 92 F F F
64542024-02-13T21:41:51.028ZMax gen: 58, Max flush: 99
64552024-02-13T21:41:51.028ZError: Difference in extent metadata found!
64562024-02-13T21:41:51.028ZA Difference in extent metadata is expected here
64572024-02-13T21:41:51.028ZOn loop 29
64582024-02-13T21:41:51.028Z
64592024-02-13T21:41:51.028ZVerifying data now
64602024-02-13T21:41:55.656ZNo change after verify
64612024-02-13T21:41:55.656ZLoop: 29 Downstairs dump after verify (and repair):
64622024-02-13T21:41:55.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64632024-02-13T21:41:55.671Z 0 000-019 52 52 52 83 83 83 F F F
64642024-02-13T21:41:55.671Z 1 020-039 51 51 51 82 82 82 F F F
64652024-02-13T21:41:55.671Z 2 040-059 54 54 54 86 86 86 F F F
64662024-02-13T21:41:55.671Z 3 060-079 58 58 58 98 98 98 F F F
64672024-02-13T21:41:55.671Z 4 080-099 54 54 54 85 85 85 F F F
64682024-02-13T21:41:55.671Z 5 100-119 59 59 59 100 100 100 F F F
64692024-02-13T21:41:55.671Z 6 120-139 57 57 57 92 92 92 F F F
64702024-02-13T21:41:55.671Z 7 140-159 54 54 54 85 85 85 F F F
64712024-02-13T21:41:55.671Z 8 160-179 53 53 53 84 84 84 F F F
64722024-02-13T21:41:55.675Z 9 180-199 57 57 57 92 92 92 F F F
64732024-02-13T21:41:55.675Z 10 200-219 58 58 58 99 99 99 F F F
64742024-02-13T21:41:55.675Z 11 220-239 50 50 50 79 79 79 F F F
64752024-02-13T21:41:55.675Z 12 240-259 59 59 59 100 100 100 F F F
64762024-02-13T21:41:55.675Z 13 260-279 58 58 58 94 94 94 F F F
64772024-02-13T21:41:55.675Z 14 280-299 58 58 58 93 93 93 F F F
64782024-02-13T21:41:55.675Z 15 300-319 58 58 58 93 93 93 F F F
64792024-02-13T21:41:55.676Z 16 320-339 57 57 57 92 92 92 F F F
64802024-02-13T21:41:55.676Z 17 340-359 56 56 56 90 90 90 F F F
64812024-02-13T21:41:55.676Z 18 360-379 58 58 58 96 96 96 F F F
64822024-02-13T21:41:55.676Z 19 380-399 58 58 58 94 94 94 F F F
64832024-02-13T21:41:55.676Z 20 400-419 53 53 53 84 84 84 F F F
64842024-02-13T21:41:55.676Z 21 420-439 58 58 58 96 96 96 F F F
64852024-02-13T21:41:55.676Z 22 440-459 57 57 57 92 92 92 F F F
64862024-02-13T21:41:55.676Z 23 460-479 58 58 58 96 96 96 F F F
64872024-02-13T21:41:55.676Z 24 480-499 58 58 58 98 98 98 F F F
64882024-02-13T21:41:55.676Z 25 500-519 55 55 55 88 88 88 F F F
64892024-02-13T21:41:55.676Z 26 520-539 54 54 54 85 85 85 F F F
64902024-02-13T21:41:55.676Z 27 540-559 58 58 58 93 93 93 F F F
64912024-02-13T21:41:55.676Z 28 560-579 54 54 54 86 86 86 F F F
64922024-02-13T21:41:55.676Z 29 580-599 57 57 57 92 92 92 F F F
64932024-02-13T21:41:55.676ZMax gen: 59, Max flush: 100
64942024-02-13T21:41:55.677Z
64952024-02-13T21:41:55.677ZBegin loop 30
64962024-02-13T21:41:55.677ZDownstairs to restart: 0
64972024-02-13T21:41:55.693Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.487626794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
64982024-02-13T21:41:55.693ZCrucible runtime is spawned
64992024-02-13T21:41:55.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.487955005Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
65002024-02-13T21:41:55.702Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.487992951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
65012024-02-13T21:41:55.702Z{"msg":"Crucible b1558d7c-e591-4865-a4b4-1a0fafa5d5b4 has session id: e8f52f56-d956-4177-b64e-6da0b55b16a5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488028168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65022024-02-13T21:41:55.702Z{"msg":"Upstairs opts: Upstairs UUID: b1558d7c-e591-4865-a4b4-1a0fafa5d5b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488075321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65032024-02-13T21:41:55.702Z{"msg":"Crucible stats registered with UUID: b1558d7c-e591-4865-a4b4-1a0fafa5d5b4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488113737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65042024-02-13T21:41:55.703Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488131431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
65052024-02-13T21:41:55.703Z{"msg":"b1558d7c-e591-4865-a4b4-1a0fafa5d5b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488143416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65062024-02-13T21:41:55.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488155722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65072024-02-13T21:41:55.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488170696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65082024-02-13T21:41:55.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.48818873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65092024-02-13T21:41:55.703Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488213871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65102024-02-13T21:41:55.703Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488251567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65112024-02-13T21:41:55.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488287954Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65122024-02-13T21:41:55.704Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488392676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65132024-02-13T21:41:55.704Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488454433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65142024-02-13T21:41:55.704Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.48849223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65152024-02-13T21:41:55.704Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.48851942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65162024-02-13T21:41:55.704Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488558796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65172024-02-13T21:41:55.704Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.488757674Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"io task","client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65182024-02-13T21:41:55.704Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.48918119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65192024-02-13T21:41:55.705Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.489215917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65202024-02-13T21:41:55.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.489231862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65212024-02-13T21:41:55.705Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.489245906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65222024-02-13T21:41:55.705Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.48976129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65232024-02-13T21:41:55.705Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.489902199Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65242024-02-13T21:41:55.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490050245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65252024-02-13T21:41:55.705Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49009205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65262024-02-13T21:41:55.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490212636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65272024-02-13T21:41:55.706Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49025726Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65282024-02-13T21:41:55.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490286729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65292024-02-13T21:41:55.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490385153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65302024-02-13T21:41:55.706Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49042116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65312024-02-13T21:41:55.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490437204Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65322024-02-13T21:41:55.706Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.490637652Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65332024-02-13T21:41:55.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491094266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65342024-02-13T21:41:55.706Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49113764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65352024-02-13T21:41:55.707Z{"msg":"[0]R flush_numbers[0..12]: [83, 82, 86, 98, 85, 100, 92, 85, 84, 92, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491155454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65362024-02-13T21:41:55.707Z{"msg":"[0]R generation[0..12]: [52, 51, 54, 58, 54, 59, 57, 54, 53, 57, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491177786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65372024-02-13T21:41:55.707Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49119249Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65382024-02-13T21:41:55.707Z{"msg":"[1]R flush_numbers[0..12]: [83, 82, 86, 98, 85, 100, 92, 85, 84, 92, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491207385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65392024-02-13T21:41:55.707Z{"msg":"[1]R generation[0..12]: [52, 51, 54, 58, 54, 59, 57, 54, 53, 57, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491227738Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65402024-02-13T21:41:55.707Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491242142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65412024-02-13T21:41:55.707Z{"msg":"Wait for a query_work_queue command to finish before sending IO
65422024-02-13T21:41:55.707Z[2]R flush_numbers[0..12]: [83, 82, 86, 98, 85, 100, 92, 85, 84, 92, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491264764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65432024-02-13T21:41:55.708Z{"msg":"[2]R generation[0..12]: [52, 51, 54, 58, 54, 59, 57, 54, 53, 57, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491279499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65442024-02-13T21:41:55.708Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491300271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65452024-02-13T21:41:55.708Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491314116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65462024-02-13T21:41:55.708Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491333929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65472024-02-13T21:41:55.708Z{"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491347104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65482024-02-13T21:41:55.708Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491360629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"":"downstairs","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65492024-02-13T21:41:55.709Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491380442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65502024-02-13T21:41:55.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491392788Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"0","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65512024-02-13T21:41:55.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491407473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"1","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65522024-02-13T21:41:55.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491427525Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"client":"2","":"downstairs","session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65532024-02-13T21:41:55.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49144114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65542024-02-13T21:41:55.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491453596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65552024-02-13T21:41:55.709Z{"msg":"b1558d7c-e591-4865-a4b4-1a0fafa5d5b4 is now active with session: e8f52f56-d956-4177-b64e-6da0b55b16a5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491467071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65562024-02-13T21:41:55.709Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.491478887Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378,"session_id":"e8f52f56-d956-4177-b64e-6da0b55b16a5"}
65572024-02-13T21:41:55.710Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:41:54.49149812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
65582024-02-13T21:41:55.710ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65592024-02-13T21:41:55.728ZLoading write count information from file "/tmp/test_repair_verify.data"
65602024-02-13T21:41:55.728ZRun Repair workload
65612024-02-13T21:41:55.729Z01/30 Flush
65622024-02-13T21:41:55.729Z02/30 Write block 378 len 36864 data: 6 6 6 8 8 7 5 4 5
65632024-02-13T21:41:55.729Z03/30 Write block 177 len 24576 data: 6 5 4 4 3 3
65642024-02-13T21:41:55.729Z04/30 Write block 313 len 8192 data: 7 6
65652024-02-13T21:41:55.729Z05/30 Read block 565 len 32768
65662024-02-13T21:41:55.729Z06/30 Write block 403 len 36864 data: 4 4 4 3 4 4 5 5 6
65672024-02-13T21:41:55.729Z07/30 Read block 384 len 28672
65682024-02-13T21:41:55.729Z08/30 Read block 387 len 16384
65692024-02-13T21:41:55.730Z09/30 Read block 98 len 8192
65702024-02-13T21:41:55.730Z10/30 Write block 497 len 4096 data: 8
65712024-02-13T21:41:55.730Z11/30 Write block 138 len 8192 data: 6 4
65722024-02-13T21:41:55.730Z12/30 Read block 235 len 40960
65732024-02-13T21:41:55.730Z13/30 Read block 246 len 40960
65742024-02-13T21:41:55.730Z14/30 Flush
65752024-02-13T21:41:55.730Z15/30 Flush
65762024-02-13T21:41:55.730Z16/30 Write block 340 len 28672 data: 7 6 6 6 4 5 5
65772024-02-13T21:41:55.730Z17/30 Flush
65782024-02-13T21:41:55.730Z18/30 Write block 159 len 12288 data: 5 4 5
65792024-02-13T21:41:55.730Z19/30 Read block 226 len 4096
65802024-02-13T21:41:55.731Z20/30 Flush
65812024-02-13T21:41:55.731Z21/30 Write block 456 len 36864 data: 5 5 6 8 7 6 6 5 7
65822024-02-13T21:41:55.731Z22/30 Read block 337 len 28672
65832024-02-13T21:41:55.739Z23/30 Write block 107 len 20480 data: 4 4 4 3 4
65842024-02-13T21:41:55.739Z24/30 Write block 38 len 36864 data: 7 6 6 5 4 3 3 3 4
65852024-02-13T21:41:55.739Z25/30 Write block 356 len 36864 data: 9 9 8 8 7 4 4 4 4
65862024-02-13T21:41:55.740Z26/30 Write block 441 len 24576 data: 6 6 5 5 6 6
65872024-02-13T21:41:55.740Z27/30 Write block 344 len 36864 data: 5 6 6 7 6 4 4 6 7
65882024-02-13T21:41:55.740Z28/30 Read block 435 len 40960
65892024-02-13T21:41:55.740Z29/30 Write block 15 len 16384 data: 8 10 9 10
65902024-02-13T21:41:55.740Z30/30 Write block 586 len 40960 data: 6 5 4 5 6 5 5 4 4 4
65912024-02-13T21:41:55.740Z----------------------------------------------------------------
65922024-02-13T21:41:55.740Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:29
65932024-02-13T21:41:55.740ZStates: Active Active Active
65942024-02-13T21:41:55.740ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65952024-02-13T21:41:55.741Z 2 Acked 1001 Write 9 Done Done Done false
65962024-02-13T21:41:55.741Z 3 Acked 1002 Write 6 Done Done Done false
65972024-02-13T21:41:55.741Z 4 Acked 1003 Write 2 Done Done Done false
65982024-02-13T21:41:55.741Z 5 Acked 1004 Read 8 Done Done Done false
65992024-02-13T21:41:55.741Z 6 Acked 1005 Write 9 Sent Done Done false
66002024-02-13T21:41:55.741Z 7 Acked 1006 Read 7 Sent Done Done false
66012024-02-13T21:41:55.741Z 8 Acked 1007 Read 4 Sent Done Done false
66022024-02-13T21:41:55.741Z 9 Acked 1008 Read 2 Sent Done Done false
66032024-02-13T21:41:55.741Z 10 Acked 1009 Write 1 Sent Done Done false
66042024-02-13T21:41:55.741Z 11 Acked 1010 Write 2 Sent Done Done false
66052024-02-13T21:41:55.741Z 12 Acked 1011 Read 10 Sent Done Done false
66062024-02-13T21:41:55.741Z 13 Acked 1012 Read 10 Sent Done Done false
66072024-02-13T21:41:55.741Z 14 Acked 1013 Flush 0 Sent Done Done false
66082024-02-13T21:41:55.741Z 15 Acked 1014 Flush 0 Sent Done Done false
66092024-02-13T21:41:55.741Z 16 Acked 1015 Write 7 Sent Done Done false
66102024-02-13T21:41:55.741Z 17 Acked 1016 Flush 0 Sent Done Done false
66112024-02-13T21:41:55.741Z 18 Acked 1017 Write 3 Sent Done Done false
66122024-02-13T21:41:55.741Z 19 Acked 1018 Read 1 Sent Done Done false
66132024-02-13T21:41:55.741Z 20 Acked 1019 Flush 0 Sent Done Done false
66142024-02-13T21:41:55.741Z 21 Acked 1020 Write 9 Sent Done Done false
66152024-02-13T21:41:55.741Z 22 Acked 1021 Read 7 Sent Done Done false
66162024-02-13T21:41:55.741Z 23 Acked 1022 Write 5 Sent Done Done false
66172024-02-13T21:41:55.741Z 24 Acked 1023 Write 9 Sent Done Done false
66182024-02-13T21:41:55.742Z 25 Acked 1024 Write 9 Sent Done Done false
66192024-02-13T21:41:55.742Z 26 Acked 1025 Write 6 Sent Done Done false
66202024-02-13T21:41:55.742Z 27 Acked 1026 Write 9 Sent Done Done false
66212024-02-13T21:41:55.742Z 28 Acked 1027 Read 10 Sent Done Done false
66222024-02-13T21:41:55.742Z 29 Acked 1028 Write 4 Sent Done Done false
66232024-02-13T21:41:55.742Z 30 Acked 1029 Write 10 Sent Sent Sent false
66242024-02-13T21:41:55.742Z STATES DS:0 DS:1 DS:2 TOTAL
66252024-02-13T21:41:55.742Z New 0 0 0 0
66262024-02-13T21:41:55.742Z Sent 25 1 1 27
66272024-02-13T21:41:55.742Z Done 4 28 28 60
66282024-02-13T21:41:55.742Z Skipped 0 0 0 0
66292024-02-13T21:41:55.742Z Error 0 0 0 0
66302024-02-13T21:41:55.742ZLast Flush: 1000 1019 1019
66312024-02-13T21:41:55.742ZDownstairs last five completed: 1000
66322024-02-13T21:41:55.742ZUpstairs last five completed: 30 29 28 27 26
66332024-02-13T21:41:55.742Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:41:54.52386291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1378}
66342024-02-13T21:41:55.742ZWrote out file "/tmp/test_repair_verify.data"
66352024-02-13T21:41:55.742Z
66362024-02-13T21:41:55.746ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66372024-02-13T21:41:55.754ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66382024-02-13T21:41:55.754Z 0 000-019 52 52 52 83 83 83 F T T <---
66392024-02-13T21:41:55.755Z 1 020-039 51 51 51 82 82 82 F T T <---
66402024-02-13T21:41:55.755Z 2 040-059 54 54 54 86 86 86 F T T <---
66412024-02-13T21:41:55.755Z 3 060-079 58 58 58 98 98 98 F F F
66422024-02-13T21:41:55.755Z 4 080-099 54 54 54 85 85 85 F F F
66432024-02-13T21:41:55.755Z 5 100-119 59 59 59 100 100 100 F T T <---
66442024-02-13T21:41:55.755Z 6 120-139 57 60 60 92 102 102 F F F <---
66452024-02-13T21:41:55.755Z 7 140-159 54 60 60 85 105 105 F F F <---
66462024-02-13T21:41:55.755Z 8 160-179 53 60 60 84 105 105 T F F <---
66472024-02-13T21:41:55.755Z 9 180-199 57 60 60 92 102 102 T F F <---
66482024-02-13T21:41:55.758Z 10 200-219 58 58 58 99 99 99 F F F
66492024-02-13T21:41:55.758Z 11 220-239 50 50 50 79 79 79 F F F
66502024-02-13T21:41:55.758Z 12 240-259 59 59 59 100 100 100 F F F
66512024-02-13T21:41:55.758Z 13 260-279 58 58 58 94 94 94 F F F
66522024-02-13T21:41:55.758Z 14 280-299 58 58 58 93 93 93 F F F
66532024-02-13T21:41:55.758Z 15 300-319 58 60 60 93 102 102 T F F <---
66542024-02-13T21:41:55.758Z 16 320-339 57 57 57 92 92 92 F F F
66552024-02-13T21:41:55.758Z 17 340-359 56 60 60 90 104 104 F T T <---
66562024-02-13T21:41:55.758Z 18 360-379 58 60 60 96 102 102 T T T <---
66572024-02-13T21:41:55.759Z 19 380-399 58 60 60 94 102 102 T F F <---
66582024-02-13T21:41:55.759Z 20 400-419 53 60 60 84 102 102 F F F <---
66592024-02-13T21:41:55.759Z 21 420-439 58 58 58 96 96 96 F F F
66602024-02-13T21:41:55.759Z 22 440-459 57 57 57 92 92 92 F T T <---
66612024-02-13T21:41:55.759Z 23 460-479 58 58 58 96 96 96 F T T <---
66622024-02-13T21:41:55.759Z 24 480-499 58 60 60 98 102 102 F F F <---
66632024-02-13T21:41:55.759Z 25 500-519 55 55 55 88 88 88 F F F
66642024-02-13T21:41:55.759Z 26 520-539 54 54 54 85 85 85 F F F
66652024-02-13T21:41:55.759Z 27 540-559 58 58 58 93 93 93 F F F
66662024-02-13T21:41:55.759Z 28 560-579 54 54 54 86 86 86 F F F
66672024-02-13T21:41:55.759Z 29 580-599 57 57 57 92 92 92 F T T <---
66682024-02-13T21:41:55.759ZMax gen: 60, Max flush: 105
66692024-02-13T21:41:55.759ZError: Difference in extent metadata found!
66702024-02-13T21:41:55.759ZA Difference in extent metadata is expected here
66712024-02-13T21:41:55.759ZOn loop 30
66722024-02-13T21:41:55.759Z
66732024-02-13T21:41:55.762ZVerifying data now
66742024-02-13T21:42:01.546ZNo change after verify
66752024-02-13T21:42:01.546ZLoop: 30 Downstairs dump after verify (and repair):
66762024-02-13T21:42:01.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66772024-02-13T21:42:01.560Z 0 000-019 61 61 61 106 106 106 F F F
66782024-02-13T21:42:01.560Z 1 020-039 61 61 61 106 106 106 F F F
66792024-02-13T21:42:01.560Z 2 040-059 61 61 61 106 106 106 F F F
66802024-02-13T21:42:01.560Z 3 060-079 58 58 58 98 98 98 F F F
66812024-02-13T21:42:01.560Z 4 080-099 54 54 54 85 85 85 F F F
66822024-02-13T21:42:01.560Z 5 100-119 61 61 61 106 106 106 F F F
66832024-02-13T21:42:01.561Z 6 120-139 60 60 60 102 102 102 F F F
66842024-02-13T21:42:01.561Z 7 140-159 60 60 60 105 105 105 F F F
66852024-02-13T21:42:01.561Z 8 160-179 60 60 60 105 105 105 F F F
66862024-02-13T21:42:01.561Z 9 180-199 60 60 60 102 102 102 F F F
66872024-02-13T21:42:01.567Z 10 200-219 58 58 58 99 99 99 F F F
66882024-02-13T21:42:01.567Z 11 220-239 50 50 50 79 79 79 F F F
66892024-02-13T21:42:01.567Z 12 240-259 59 59 59 100 100 100 F F F
66902024-02-13T21:42:01.567Z 13 260-279 58 58 58 94 94 94 F F F
66912024-02-13T21:42:01.567Z 14 280-299 58 58 58 93 93 93 F F F
66922024-02-13T21:42:01.567Z 15 300-319 60 60 60 102 102 102 F F F
66932024-02-13T21:42:01.567Z 16 320-339 57 57 57 92 92 92 F F F
66942024-02-13T21:42:01.567Z 17 340-359 61 61 61 106 106 106 F F F
66952024-02-13T21:42:01.567Z 18 360-379 61 61 61 106 106 106 F F F
66962024-02-13T21:42:01.568Z 19 380-399 60 60 60 102 102 102 F F F
66972024-02-13T21:42:01.568Z 20 400-419 60 60 60 102 102 102 F F F
66982024-02-13T21:42:01.568Z 21 420-439 58 58 58 96 96 96 F F F
66992024-02-13T21:42:01.568Z 22 440-459 61 61 61 106 106 106 F F F
67002024-02-13T21:42:01.568Z 23 460-479 61 61 61 106 106 106 F F F
67012024-02-13T21:42:01.568Z 24 480-499 60 60 60 102 102 102 F F F
67022024-02-13T21:42:01.568Z 25 500-519 55 55 55 88 88 88 F F F
67032024-02-13T21:42:01.568Z 26 520-539 54 54 54 85 85 85 F F F
67042024-02-13T21:42:01.568Z 27 540-559 58 58 58 93 93 93 F F F
67052024-02-13T21:42:01.568Z 28 560-579 54 54 54 86 86 86 F F F
67062024-02-13T21:42:01.568Z 29 580-599 61 61 61 106 106 106 F F F
67072024-02-13T21:42:01.568ZMax gen: 61, Max flush: 106
67082024-02-13T21:42:01.568Z
67092024-02-13T21:42:01.568ZBegin loop 31
67102024-02-13T21:42:01.568ZDownstairs to restart: 1
67112024-02-13T21:42:01.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374038031Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
67122024-02-13T21:42:01.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
67132024-02-13T21:42:01.595Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374366661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
67142024-02-13T21:42:01.595Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374405138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
67152024-02-13T21:42:01.595Z{"msg":"Crucible 637112a5-f142-4a38-9466-e50ce326de9d has session id: 3a8a2e99-5edc-4ae7-827c-561fce821c8a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374429179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67162024-02-13T21:42:01.595Z{"msg":"Upstairs opts: Upstairs UUID: 637112a5-f142-4a38-9466-e50ce326de9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374471773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67172024-02-13T21:42:01.595Z{"msg":"Crucible stats registered with UUID: 637112a5-f142-4a38-9466-e50ce326de9d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374488757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67182024-02-13T21:42:01.596Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
67192024-02-13T21:42:01.596Z{"msg":"637112a5-f142-4a38-9466-e50ce326de9d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374522705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67202024-02-13T21:42:01.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37453543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67212024-02-13T21:42:01.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374558282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67222024-02-13T21:42:01.596Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374572517Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67232024-02-13T21:42:01.596Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374586961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67242024-02-13T21:42:01.597Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374606205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67252024-02-13T21:42:01.597Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374630976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67262024-02-13T21:42:01.597Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374655317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67272024-02-13T21:42:01.597Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374676029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67282024-02-13T21:42:01.597Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.3747003Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67292024-02-13T21:42:01.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.374824575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67302024-02-13T21:42:01.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.375025173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67312024-02-13T21:42:01.598Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.375081222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"io task","client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67322024-02-13T21:42:01.598Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37547261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67332024-02-13T21:42:01.598Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.375530249Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67342024-02-13T21:42:01.598Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.375706006Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67352024-02-13T21:42:01.598Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.375746841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67362024-02-13T21:42:01.598Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376105161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67372024-02-13T21:42:01.599Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376220569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67382024-02-13T21:42:01.599Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376572951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67392024-02-13T21:42:01.599Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376643466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67402024-02-13T21:42:01.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37674117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67412024-02-13T21:42:01.599Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376771879Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67422024-02-13T21:42:01.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376798519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67432024-02-13T21:42:01.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376828858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67442024-02-13T21:42:01.600Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.376849581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67452024-02-13T21:42:01.600Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37687813Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67462024-02-13T21:42:01.600Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377119323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67472024-02-13T21:42:01.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377557444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67482024-02-13T21:42:01.600Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377601638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67492024-02-13T21:42:01.600Z{"msg":"[0]R flush_numbers[0..12]: [106, 106, 106, 98, 85, 106, 102, 105, 105, 102, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377620681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67502024-02-13T21:42:01.600Z{"msg":"[0]R generation[0..12]: [61, 61, 61, 58, 54, 61, 60, 60, 60, 60, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377642873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67512024-02-13T21:42:01.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377659027Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67522024-02-13T21:42:01.600Z{"msg":"[1]R flush_numbers[0..12]: [106, 106, 106, 98, 85, 106, 102, 105, 105, 102, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377680179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67532024-02-13T21:42:01.600Z{"msg":"[1]R generation[0..12]: [61, 61, 61, 58, 54, 61, 60, 60, 60, 60, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377694204Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67542024-02-13T21:42:01.600Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377715247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67552024-02-13T21:42:01.600Z{"msg":"[2]R flush_numbers[0..12]: [106, 106, 106, 98, 85, 106, 102, 105, 105, 102, 99, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377729972Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67562024-02-13T21:42:01.601ZWait for a query_work_queue command to finish before sending IO
67572024-02-13T21:42:01.601Z{"msg":"[2]R generation[0..12]: [61, 61, 61, 58, 54, 61, 60, 60, 60, 60, 58, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377754272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67582024-02-13T21:42:01.601Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377769297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67592024-02-13T21:42:01.601Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377783432Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67602024-02-13T21:42:01.601Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377797277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67612024-02-13T21:42:01.601Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377810442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67622024-02-13T21:42:01.601Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377830395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"":"downstairs","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67632024-02-13T21:42:01.601Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37784514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67642024-02-13T21:42:01.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377857525Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"0","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67652024-02-13T21:42:01.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.37787101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"1","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67662024-02-13T21:42:01.601Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377891173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"client":"2","":"downstairs","session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67672024-02-13T21:42:01.601Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377904848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67682024-02-13T21:42:01.601Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377917623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67692024-02-13T21:42:01.601Z{"msg":"637112a5-f142-4a38-9466-e50ce326de9d is now active with session: 3a8a2e99-5edc-4ae7-827c-561fce821c8a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377937236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67702024-02-13T21:42:01.601Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377949262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386,"session_id":"3a8a2e99-5edc-4ae7-827c-561fce821c8a"}
67712024-02-13T21:42:01.601Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:00.377961577Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
67722024-02-13T21:42:01.601ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67732024-02-13T21:42:01.601ZLoading write count information from file "/tmp/test_repair_verify.data"
67742024-02-13T21:42:01.601ZRun Repair workload
67752024-02-13T21:42:01.601Z01/30 Write block 95 len 36864 data: 4 4 5 6 6 6 7 11 10
67762024-02-13T21:42:01.601Z02/30 Read block 130 len 40960
67772024-02-13T21:42:01.606Z03/30 Read block 486 len 40960
67782024-02-13T21:42:01.607Z04/30 Write block 10 len 32768 data: 6 8 7 7 8 9 11 10
67792024-02-13T21:42:01.607Z05/30 Write block 277 len 20480 data: 8 8 8 7 3
67802024-02-13T21:42:01.607Z06/30 Write block 298 len 32768 data: 6 5 5 4 4 4 4 4
67812024-02-13T21:42:01.607Z07/30 Write block 420 len 40960 data: 7 6 5 5 4 5 6 5 6 8
67822024-02-13T21:42:01.607Z08/30 Write block 81 len 4096 data: 7
67832024-02-13T21:42:01.607Z09/30 Read block 430 len 4096
67842024-02-13T21:42:01.607Z10/30 Read block 103 len 24576
67852024-02-13T21:42:01.608Z11/30 Write block 67 len 12288 data: 3 6 7
67862024-02-13T21:42:01.608Z12/30 Write block 23 len 8192 data: 3 3
67872024-02-13T21:42:01.608Z13/30 Write block 113 len 4096 data: 3
67882024-02-13T21:42:01.608Z14/30 Write block 208 len 28672 data: 4 5 6 5 5 6 6
67892024-02-13T21:42:01.608Z15/30 Write block 415 len 40960 data: 6 7 8 8 8 8 7 6 6 5
67902024-02-13T21:42:01.608Z16/30 Read block 152 len 4096
67912024-02-13T21:42:01.608Z17/30 Write block 137 len 24576 data: 5 7 5 5 6 7
67922024-02-13T21:42:01.608Z18/30 Read block 231 len 20480
67932024-02-13T21:42:01.608Z19/30 Read block 591 len 16384
67942024-02-13T21:42:01.609Z20/30 Read block 502 len 24576
67952024-02-13T21:42:01.609Z21/30 Write block 215 len 24576 data: 6 8 9 9 8 5
67962024-02-13T21:42:01.609Z22/30 Write block 194 len 24576 data: 8 7 7 7 8 8
67972024-02-13T21:42:01.609Z23/30 Write block 187 len 16384 data: 6 8 8 8
67982024-02-13T21:42:01.609Z24/30 Write block 288 len 28672 data: 4 5 6 6 7 8 8
67992024-02-13T21:42:01.616Z25/30 Write block 59 len 4096 data: 7
68002024-02-13T21:42:01.616Z26/30 Read block 21 len 36864
68012024-02-13T21:42:01.616Z27/30 Write block 287 len 12288 data: 3 5 6
68022024-02-13T21:42:01.616Z28/30 Write block 172 len 4096 data: 6
68032024-02-13T21:42:01.616Z29/30 Read block 189 len 32768
68042024-02-13T21:42:01.616Z30/30 Write block 432 len 36864 data: 9 6 7 7 5 5 6 6 5
68052024-02-13T21:42:01.616Z----------------------------------------------------------------
68062024-02-13T21:42:01.617Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
68072024-02-13T21:42:01.617ZStates: Active Active Active
68082024-02-13T21:42:01.617ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
68092024-02-13T21:42:01.617Z 1 Acked 1000 Write 9 Done Sent Done false
68102024-02-13T21:42:01.617Z 2 Acked 1001 Read 10 Done Sent Done false
68112024-02-13T21:42:01.617Z 3 Acked 1002 Read 10 Done Sent Done false
68122024-02-13T21:42:01.617Z 4 Acked 1003 Write 8 Done Sent Done false
68132024-02-13T21:42:01.617Z 5 Acked 1004 Write 5 Done Sent Done false
68142024-02-13T21:42:01.617Z 6 Acked 1005 Write 8 Done Sent Done false
68152024-02-13T21:42:01.618Z 7 Acked 1006 Write 10 Done Sent Done false
68162024-02-13T21:42:01.618Z 8 Acked 1007 Write 1 Done Sent Done false
68172024-02-13T21:42:01.618Z 9 Acked 1008 Read 1 Done Sent Done false
68182024-02-13T21:42:01.618Z 10 Acked 1009 Read 6 Done Sent Done false
68192024-02-13T21:42:01.618Z 11 Acked 1010 Write 3 Done Sent Done false
68202024-02-13T21:42:01.618Z 12 Acked 1011 Write 2 Done Sent Done false
68212024-02-13T21:42:01.618Z 13 Acked 1012 Write 1 Done Sent Done false
68222024-02-13T21:42:01.618Z 14 Acked 1013 Write 7 Done Sent Done false
68232024-02-13T21:42:01.619Z 15 Acked 1014 Write 10 Done Sent Done false
68242024-02-13T21:42:01.619Z 16 Acked 1015 Read 1 Done Sent Done false
68252024-02-13T21:42:01.619Z 17 Acked 1016 Write 6 Done Sent Done false
68262024-02-13T21:42:01.619Z 18 Acked 1017 Read 5 Done Sent Done false
68272024-02-13T21:42:01.619Z 19 Acked 1018 Read 4 Done Sent Done false
68282024-02-13T21:42:01.619Z 20 Acked 1019 Read 6 Done Sent Done false
68292024-02-13T21:42:01.619Z 21 Acked 1020 Write 6 Done Sent Done false
68302024-02-13T21:42:01.620Z 22 Acked 1021 Write 6 Done Sent Done false
68312024-02-13T21:42:01.620Z 23 Acked 1022 Write 4 Done Sent Done false
68322024-02-13T21:42:01.620Z 24 Acked 1023 Write 7 Done Sent Done false
68332024-02-13T21:42:01.620Z 25 Acked 1024 Write 1 Done Sent Done false
68342024-02-13T21:42:01.620Z 26 Acked 1025 Read 9 Done Sent Done false
68352024-02-13T21:42:01.620Z 27 Acked 1026 Write 3 Done Sent Done false
68362024-02-13T21:42:01.621Z 28 Acked 1027 Write 1 Done Sent Done false
68372024-02-13T21:42:01.621Z 29 Acked 1028 Read 8 Done Sent Done false
68382024-02-13T21:42:01.621Z 30 Acked 1029 Write 9 Sent Sent Sent false
68392024-02-13T21:42:01.621Z STATES DS:0 DS:1 DS:2 TOTAL
68402024-02-13T21:42:01.621Z New 0 0 0 0
68412024-02-13T21:42:01.621Z Sent 1 30 1 32
68422024-02-13T21:42:01.621Z Done 29 0 29 58
68432024-02-13T21:42:01.622Z Skipped 0 0 0 0
68442024-02-13T21:42:01.622Z Error 0 0 0 0
68452024-02-13T21:42:01.622ZLast Flush: 0 0 0
68462024-02-13T21:42:01.622ZDownstairs last five completed:
68472024-02-13T21:42:01.622ZUpstairs last five completed: 30 29 28 27 26
68482024-02-13T21:42:01.622Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:00.403656127Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1386}
68492024-02-13T21:42:01.623ZWrote out file "/tmp/test_repair_verify.data"
68502024-02-13T21:42:01.628Z
68512024-02-13T21:42:01.628ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68522024-02-13T21:42:01.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68532024-02-13T21:42:01.639Z 0 000-019 61 61 61 106 106 106 T F T <---
68542024-02-13T21:42:01.639Z 1 020-039 61 61 61 106 106 106 T F T <---
68552024-02-13T21:42:01.639Z 2 040-059 61 61 61 106 106 106 T F T <---
68562024-02-13T21:42:01.640Z 3 060-079 58 58 58 98 98 98 T F T <---
68572024-02-13T21:42:01.640Z 4 080-099 54 54 54 85 85 85 T F T <---
68582024-02-13T21:42:01.640Z 5 100-119 61 61 61 106 106 106 T F T <---
68592024-02-13T21:42:01.640Z 6 120-139 60 60 60 102 102 102 T F T <---
68602024-02-13T21:42:01.640Z 7 140-159 60 60 60 105 105 105 T F T <---
68612024-02-13T21:42:01.640Z 8 160-179 60 60 60 105 105 105 T F T <---
68622024-02-13T21:42:01.645Z 9 180-199 60 60 60 102 102 102 T F T <---
68632024-02-13T21:42:01.645Z 10 200-219 58 58 58 99 99 99 T F T <---
68642024-02-13T21:42:01.645Z 11 220-239 50 50 50 79 79 79 T F T <---
68652024-02-13T21:42:01.645Z 12 240-259 59 59 59 100 100 100 F F F
68662024-02-13T21:42:01.645Z 13 260-279 58 58 58 94 94 94 T F T <---
68672024-02-13T21:42:01.646Z 14 280-299 58 58 58 93 93 93 T F T <---
68682024-02-13T21:42:01.646Z 15 300-319 60 60 60 102 102 102 T F T <---
68692024-02-13T21:42:01.646Z 16 320-339 57 57 57 92 92 92 F F F
68702024-02-13T21:42:01.646Z 17 340-359 61 61 61 106 106 106 F F F
68712024-02-13T21:42:01.646Z 18 360-379 61 61 61 106 106 106 F F F
68722024-02-13T21:42:01.646Z 19 380-399 60 60 60 102 102 102 F F F
68732024-02-13T21:42:01.647Z 20 400-419 60 60 60 102 102 102 T F T <---
68742024-02-13T21:42:01.647Z 21 420-439 58 58 58 96 96 96 T F T <---
68752024-02-13T21:42:01.647Z 22 440-459 61 61 61 106 106 106 T F T <---
68762024-02-13T21:42:01.647Z 23 460-479 61 61 61 106 106 106 F F F
68772024-02-13T21:42:01.647Z 24 480-499 60 60 60 102 102 102 F F F
68782024-02-13T21:42:01.648Z 25 500-519 55 55 55 88 88 88 F F F
68792024-02-13T21:42:01.648Z 26 520-539 54 54 54 85 85 85 F F F
68802024-02-13T21:42:01.648Z 27 540-559 58 58 58 93 93 93 F F F
68812024-02-13T21:42:01.648Z 28 560-579 54 54 54 86 86 86 F F F
68822024-02-13T21:42:01.648Z 29 580-599 61 61 61 106 106 106 F F F
68832024-02-13T21:42:01.648ZMax gen: 61, Max flush: 106
68842024-02-13T21:42:01.648ZError: Difference in extent metadata found!
68852024-02-13T21:42:01.649ZA Difference in extent metadata is expected here
68862024-02-13T21:42:01.649ZOn loop 31
68872024-02-13T21:42:01.649Z
68882024-02-13T21:42:01.654ZVerifying data now
68892024-02-13T21:42:06.519ZNo change after verify
68902024-02-13T21:42:06.520ZLoop: 31 Downstairs dump after verify (and repair):
68912024-02-13T21:42:07.343ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68922024-02-13T21:42:07.343Z 0 000-019 62 62 62 107 107 107 F F F
68932024-02-13T21:42:07.344Z 1 020-039 62 62 62 107 107 107 F F F
68942024-02-13T21:42:07.344Z 2 040-059 62 62 62 107 107 107 F F F
68952024-02-13T21:42:07.344Z 3 060-079 62 62 62 107 107 107 F F F
68962024-02-13T21:42:07.344Z 4 080-099 62 62 62 107 107 107 F F F
68972024-02-13T21:42:07.344Z 5 100-119 62 62 62 107 107 107 F F F
68982024-02-13T21:42:07.345Z 6 120-139 62 62 62 107 107 107 F F F
68992024-02-13T21:42:07.345Z 7 140-159 62 62 62 107 107 107 F F F
69002024-02-13T21:42:07.345Z 8 160-179 62 62 62 107 107 107 F F F
69012024-02-13T21:42:07.345Z 9 180-199 62 62 62 107 107 107 F F F
69022024-02-13T21:42:07.345Z 10 200-219 62 62 62 107 107 107 F F F
69032024-02-13T21:42:07.345Z 11 220-239 62 62 62 107 107 107 F F F
69042024-02-13T21:42:07.345Z 12 240-259 59 59 59 100 100 100 F F F
69052024-02-13T21:42:07.346Z 13 260-279 62 62 62 107 107 107 F F F
69062024-02-13T21:42:07.346Z 14 280-299 62 62 62 107 107 107 F F F
69072024-02-13T21:42:07.346Z 15 300-319 62 62 62 107 107 107 F F F
69082024-02-13T21:42:07.346Z 16 320-339 57 57 57 92 92 92 F F F
69092024-02-13T21:42:07.346Z 17 340-359 61 61 61 106 106 106 F F F
69102024-02-13T21:42:07.346Z 18 360-379 61 61 61 106 106 106 F F F
69112024-02-13T21:42:07.346Z 19 380-399 60 60 60 102 102 102 F F F
69122024-02-13T21:42:07.346Z 20 400-419 62 62 62 107 107 107 F F F
69132024-02-13T21:42:07.346Z 21 420-439 62 62 62 107 107 107 F F F
69142024-02-13T21:42:07.347Z 22 440-459 62 62 62 107 107 107 F F F
69152024-02-13T21:42:07.347Z 23 460-479 61 61 61 106 106 106 F F F
69162024-02-13T21:42:07.347Z 24 480-499 60 60 60 102 102 102 F F F
69172024-02-13T21:42:07.347Z 25 500-519 55 55 55 88 88 88 F F F
69182024-02-13T21:42:07.347Z 26 520-539 54 54 54 85 85 85 F F F
69192024-02-13T21:42:07.347Z 27 540-559 58 58 58 93 93 93 F F F
69202024-02-13T21:42:07.348Z 28 560-579 54 54 54 86 86 86 F F F
69212024-02-13T21:42:07.348Z 29 580-599 61 61 61 106 106 106 F F F
69222024-02-13T21:42:07.348ZMax gen: 62, Max flush: 107
69232024-02-13T21:42:07.348Z
69242024-02-13T21:42:07.348ZBegin loop 32
69252024-02-13T21:42:07.348ZDownstairs to restart: 1
69262024-02-13T21:42:07.348Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.3393587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
69272024-02-13T21:42:07.349Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339659171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
69282024-02-13T21:42:07.349ZCrucible runtime is spawned
69292024-02-13T21:42:07.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339715201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
69302024-02-13T21:42:07.349Z{"msg":"Crucible 9972e02c-6642-43d1-b3b1-85645861414d has session id: bcb379e0-437d-44d7-90de-57fff45af9a1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339756575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69312024-02-13T21:42:07.349Z{"msg":"Upstairs opts: Upstairs UUID: 9972e02c-6642-43d1-b3b1-85645861414d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339786564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69322024-02-13T21:42:07.349Z{"msg":"Crucible stats registered with UUID: 9972e02c-6642-43d1-b3b1-85645861414d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.33982816Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69332024-02-13T21:42:07.349Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339871214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
69342024-02-13T21:42:07.349Z{"msg":"9972e02c-6642-43d1-b3b1-85645861414d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339920676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69352024-02-13T21:42:07.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.339956313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69362024-02-13T21:42:07.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340000697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69372024-02-13T21:42:07.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340035304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69382024-02-13T21:42:07.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340067503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69392024-02-13T21:42:07.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340110217Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69402024-02-13T21:42:07.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340198935Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69412024-02-13T21:42:07.350Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340247747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69422024-02-13T21:42:07.350Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340312724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69432024-02-13T21:42:07.351Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340339494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69442024-02-13T21:42:07.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340368024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69452024-02-13T21:42:07.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340412437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69462024-02-13T21:42:07.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.340439728Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"io task","client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69472024-02-13T21:42:07.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341098159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69482024-02-13T21:42:07.351Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341145302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69492024-02-13T21:42:07.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341175501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69502024-02-13T21:42:07.351Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341213777Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69512024-02-13T21:42:07.352Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341725581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69522024-02-13T21:42:07.352Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.341916862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69532024-02-13T21:42:07.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342071446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69542024-02-13T21:42:07.353Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342101205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69552024-02-13T21:42:07.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342275122Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69562024-02-13T21:42:07.353Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.34230763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69572024-02-13T21:42:07.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342333111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69582024-02-13T21:42:07.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342376135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69592024-02-13T21:42:07.353Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342400626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69602024-02-13T21:42:07.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.342414631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69612024-02-13T21:42:07.354Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.3426121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69622024-02-13T21:42:07.354Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343041774Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69632024-02-13T21:42:07.354Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.34307933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69642024-02-13T21:42:07.354Z{"msg":"[0]R flush_numbers[0..12]: [107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343096284Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69652024-02-13T21:42:07.354Z{"msg":"[0]R generation[0..12]: [62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343123564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69662024-02-13T21:42:07.355Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343150474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69672024-02-13T21:42:07.355Z{"msg":"[1]R flush_numbers[0..12]: [107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343165579Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69682024-02-13T21:42:07.355Z{"msg":"[1]R generation[0..12]: [62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343186641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69692024-02-13T21:42:07.355Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343201246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69702024-02-13T21:42:07.355Z{"msg":"[2]R flush_numbers[0..12]: [107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343215561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69712024-02-13T21:42:07.355Z{"msg":"[2]R generation[0..12]: [62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343236173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69722024-02-13T21:42:07.355Z{Wait for a query_work_queue command to finish before sending IO
69732024-02-13T21:42:07.355Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343256326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69742024-02-13T21:42:07.356Z{"msg":"Max found gen is 63","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343277978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69752024-02-13T21:42:07.356Z{"msg":"Generation requested: 64 >= found:63","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343291753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69762024-02-13T21:42:07.356Z{"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343318303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69772024-02-13T21:42:07.356Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343331849Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"":"downstairs","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69782024-02-13T21:42:07.356Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343345384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69792024-02-13T21:42:07.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343358319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"0","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69802024-02-13T21:42:07.357Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343371644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"1","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69812024-02-13T21:42:07.357Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343396975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"client":"2","":"downstairs","session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69822024-02-13T21:42:07.357Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.34341182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69832024-02-13T21:42:07.357Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343432212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69842024-02-13T21:42:07.357Z{"msg":"9972e02c-6642-43d1-b3b1-85645861414d is now active with session: bcb379e0-437d-44d7-90de-57fff45af9a1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343445667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69852024-02-13T21:42:07.358Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343458923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394,"session_id":"bcb379e0-437d-44d7-90de-57fff45af9a1"}
69862024-02-13T21:42:07.358Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:05.343477985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
69872024-02-13T21:42:07.358ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69882024-02-13T21:42:07.358ZLoading write count information from file "/tmp/test_repair_verify.data"
69892024-02-13T21:42:07.358ZRun Repair workload
69902024-02-13T21:42:07.359Z01/30 Write block 101 len 36864 data: 8 12 11 9 8 6 5 5 5
69912024-02-13T21:42:08.176Z02/30 Write block 489 len 16384 data: 5 3 4 4
69922024-02-13T21:42:08.176Z03/30 Read block 107 len 36864
69932024-02-13T21:42:08.176Z04/30 Flush
69942024-02-13T21:42:08.176Z05/30 Write block 197 len 28672 data: 8 9 9 6 6 6 6
69952024-02-13T21:42:08.176Z06/30 Read block 254 len 28672
69962024-02-13T21:42:08.176Z07/30 Write block 307 len 40960 data: 6 5 4 5 7 9 8 7 6 6
69972024-02-13T21:42:08.176Z08/30 Write block 86 len 8192 data: 6 5
69982024-02-13T21:42:08.176Z09/30 Read block 304 len 20480
69992024-02-13T21:42:08.176Z10/30 Flush
70002024-02-13T21:42:08.176Z11/30 Write block 238 len 12288 data: 8 8 8
70012024-02-13T21:42:08.176Z12/30 Write block 365 len 40960 data: 3 5 6 5 5 5 5 5 6 6
70022024-02-13T21:42:08.176Z13/30 Write block 533 len 20480 data: 3 4 4 4 4
70032024-02-13T21:42:08.176Z14/30 Write block 112 len 32768 data: 4 4 4 4 6 6 6 6
70042024-02-13T21:42:08.177Z15/30 Read block 469 len 40960
70052024-02-13T21:42:08.177Z16/30 Read block 266 len 4096
70062024-02-13T21:42:08.177Z17/30 Read block 77 len 12288
70072024-02-13T21:42:08.177Z18/30 Write block 34 len 12288 data: 8 9 9
70082024-02-13T21:42:08.177Z19/30 Write block 528 len 32768 data: 4 3 3 2 2 4 5 5
70092024-02-13T21:42:08.177Z20/30 Flush
70102024-02-13T21:42:08.177Z21/30 Write block 196 len 20480 data: 8 9 10 10 7
70112024-02-13T21:42:08.177Z22/30 Write block 35 len 28672 data: 10 10 7 8 7 7 6
70122024-02-13T21:42:08.177Z23/30 Read block 545 len 12288
70132024-02-13T21:42:08.177Z24/30 Read block 543 len 28672
70142024-02-13T21:42:08.177Z25/30 Read block 427 len 4096
70152024-02-13T21:42:08.177Z26/30 Write block 393 len 8192 data: 5 4
70162024-02-13T21:42:08.177Z27/30 Write block 207 len 24576 data: 3 5 6 7 6 6
70172024-02-13T21:42:08.177Z28/30 Write block 271 len 28672 data: 5 5 5 5 7 7 9
70182024-02-13T21:42:08.177Z29/30 Write block 254 len 8192 data: 6 6
70192024-02-13T21:42:08.177Z30/30 Read block 317 len 4096
70202024-02-13T21:42:08.177Z----------------------------------------------------------------
70212024-02-13T21:42:08.177Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
70222024-02-13T21:42:08.177ZStates: Active Active Active
70232024-02-13T21:42:08.177ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70242024-02-13T21:42:08.177Z 1 Acked 1000 Write 9 Done Done Done false
70252024-02-13T21:42:08.177Z 2 Acked 1001 Write 4 Done Done Done false
70262024-02-13T21:42:08.177Z 3 Acked 1002 Read 9 Done Done Done false
70272024-02-13T21:42:08.177Z 4 Acked 1003 Flush 0 Done Sent Done false
70282024-02-13T21:42:08.178Z 5 Acked 1004 Write 7 Done Sent Done false
70292024-02-13T21:42:08.178Z 6 Acked 1005 Read 7 Done Sent Done false
70302024-02-13T21:42:08.178Z 7 Acked 1006 Write 10 Done Sent Done false
70312024-02-13T21:42:08.178Z 8 Acked 1007 Write 2 Done Sent Done false
70322024-02-13T21:42:08.178Z 9 Acked 1008 Read 5 Done Sent Done false
70332024-02-13T21:42:08.178Z 10 Acked 1009 Flush 0 Done Sent Done false
70342024-02-13T21:42:08.178Z 11 Acked 1010 Write 3 Done Sent Done false
70352024-02-13T21:42:08.178Z 12 Acked 1011 Write 10 Done Sent Done false
70362024-02-13T21:42:08.178Z 13 Acked 1012 Write 5 Done Sent Done false
70372024-02-13T21:42:08.178Z 14 Acked 1013 Write 8 Done Sent Done false
70382024-02-13T21:42:08.178Z 15 Acked 1014 Read 10 Done Sent Done false
70392024-02-13T21:42:08.178Z 16 Acked 1015 Read 1 Done Sent Done false
70402024-02-13T21:42:08.178Z 17 Acked 1016 Read 3 Done Sent Done false
70412024-02-13T21:42:08.178Z 18 Acked 1017 Write 3 Done Sent Done false
70422024-02-13T21:42:08.178Z 19 Acked 1018 Write 8 Done Sent Done false
70432024-02-13T21:42:08.178Z 20 Acked 1019 Flush 0 Done Sent Done false
70442024-02-13T21:42:08.178Z 21 Acked 1020 Write 5 Done Sent Done false
70452024-02-13T21:42:08.178Z 22 Acked 1021 Write 7 Done Sent Done false
70462024-02-13T21:42:08.178Z 23 Acked 1022 Read 3 Done Sent Done false
70472024-02-13T21:42:08.178Z 24 Acked 1023 Read 7 Done Sent Done false
70482024-02-13T21:42:08.179Z 25 Acked 1024 Read 1 Done Sent Done false
70492024-02-13T21:42:08.179Z 26 Acked 1025 Write 2 Done Sent Done false
70502024-02-13T21:42:08.179Z 27 Acked 1026 Write 6 Done Sent Done false
70512024-02-13T21:42:08.179Z 28 Acked 1027 Write 7 Done Sent Done false
70522024-02-13T21:42:08.179Z 29 Acked 1028 Write 2 Done Sent Done false
70532024-02-13T21:42:08.179Z 30 Acked 1029 Read 1 Done Sent Sent false
70542024-02-13T21:42:08.179Z STATES DS:0 DS:1 DS:2 TOTAL
70552024-02-13T21:42:08.179Z New 0 0 0 0
70562024-02-13T21:42:08.179Z Sent 0 27 1 28
70572024-02-13T21:42:08.179Z Done 30 3 29 62
70582024-02-13T21:42:08.179Z Skipped 0 0 0 0
70592024-02-13T21:42:08.179Z Error 0 0 0 0
70602024-02-13T21:42:08.179ZLast Flush: 1019 0 1019
70612024-02-13T21:42:08.179ZDownstairs last five completed:
70622024-02-13T21:42:08.179ZUpstairs last five completed: 30 29 28 27 26
70632024-02-13T21:42:08.179Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:05.37671338Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1394}
70642024-02-13T21:42:08.179ZWrote out file "/tmp/test_repair_verify.data"
70652024-02-13T21:42:08.179Z
70662024-02-13T21:42:08.179ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70672024-02-13T21:42:08.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70682024-02-13T21:42:08.179Z 0 000-019 62 62 62 107 107 107 F F F
70692024-02-13T21:42:08.179Z 1 020-039 64 62 64 110 107 110 T F T <---
70702024-02-13T21:42:08.179Z 2 040-059 62 62 62 107 107 107 T F T <---
70712024-02-13T21:42:08.180Z 3 060-079 62 62 62 107 107 107 F F F
70722024-02-13T21:42:08.180Z 4 080-099 64 62 64 109 107 109 F F F <---
70732024-02-13T21:42:08.180Z 5 100-119 64 62 64 110 107 110 F T F <---
70742024-02-13T21:42:08.180Z 6 120-139 62 62 62 107 107 107 F F F
70752024-02-13T21:42:08.180Z 7 140-159 62 62 62 107 107 107 F F F
70762024-02-13T21:42:08.180Z 8 160-179 62 62 62 107 107 107 F F F
70772024-02-13T21:42:08.180Z 9 180-199 64 62 64 109 107 109 T F T <---
70782024-02-13T21:42:08.180Z 10 200-219 64 62 64 109 107 109 T F T <---
70792024-02-13T21:42:08.180Z 11 220-239 64 62 64 110 107 110 F F F <---
70802024-02-13T21:42:08.180Z 12 240-259 64 59 64 110 100 110 T F T <---
70812024-02-13T21:42:08.180Z 13 260-279 62 62 62 107 107 107 T F T <---
70822024-02-13T21:42:08.180Z 14 280-299 62 62 62 107 107 107 F F F
70832024-02-13T21:42:08.180Z 15 300-319 64 62 64 109 107 109 F F F <---
70842024-02-13T21:42:08.180Z 16 320-339 57 57 57 92 92 92 F F F
70852024-02-13T21:42:08.180Z 17 340-359 61 61 61 106 106 106 F F F
70862024-02-13T21:42:08.180Z 18 360-379 64 61 64 110 106 110 F F F <---
70872024-02-13T21:42:08.180Z 19 380-399 60 60 60 102 102 102 T F T <---
70882024-02-13T21:42:08.180Z 20 400-419 62 62 62 107 107 107 F F F
70892024-02-13T21:42:08.180Z 21 420-439 62 62 62 107 107 107 F F F
70902024-02-13T21:42:08.180Z 22 440-459 62 62 62 107 107 107 F F F
70912024-02-13T21:42:08.188Z 23 460-479 61 61 61 106 106 106 F F F
70922024-02-13T21:42:08.188Z 24 480-499 64 60 64 108 102 108 F T F <---
70932024-02-13T21:42:08.188Z 25 500-519 55 55 55 88 88 88 F F F
70942024-02-13T21:42:08.188Z 26 520-539 64 54 64 110 85 110 F F F <---
70952024-02-13T21:42:08.188Z 27 540-559 58 58 58 93 93 93 F F F
70962024-02-13T21:42:08.188Z 28 560-579 54 54 54 86 86 86 F F F
70972024-02-13T21:42:08.188Z 29 580-599 61 61 61 106 106 106 F F F
70982024-02-13T21:42:08.188ZMax gen: 64, Max flush: 110
70992024-02-13T21:42:08.188ZError: Difference in extent metadata found!
71002024-02-13T21:42:08.188ZA Difference in extent metadata is expected here
71012024-02-13T21:42:08.188ZOn loop 32
71022024-02-13T21:42:08.188Z
71032024-02-13T21:42:08.188ZVerifying data now
71042024-02-13T21:42:12.302ZNo change after verify
71052024-02-13T21:42:12.302ZLoop: 32 Downstairs dump after verify (and repair):
71062024-02-13T21:42:12.317ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
71072024-02-13T21:42:12.317Z 0 000-019 62 62 62 107 107 107 F F F
71082024-02-13T21:42:12.317Z 1 020-039 65 65 65 111 111 111 F F F
71092024-02-13T21:42:12.317Z 2 040-059 65 65 65 111 111 111 F F F
71102024-02-13T21:42:12.317Z 3 060-079 62 62 62 107 107 107 F F F
71112024-02-13T21:42:12.317Z 4 080-099 64 64 64 109 109 109 F F F
71122024-02-13T21:42:12.323Z 5 100-119 64 64 64 110 110 110 F F F
71132024-02-13T21:42:12.323Z 6 120-139 62 62 62 107 107 107 F F F
71142024-02-13T21:42:12.323Z 7 140-159 62 62 62 107 107 107 F F F
71152024-02-13T21:42:12.323Z 8 160-179 62 62 62 107 107 107 F F F
71162024-02-13T21:42:12.323Z 9 180-199 65 65 65 111 111 111 F F F
71172024-02-13T21:42:12.323Z 10 200-219 65 65 65 111 111 111 F F F
71182024-02-13T21:42:12.323Z 11 220-239 64 64 64 110 110 110 F F F
71192024-02-13T21:42:12.323Z 12 240-259 65 65 65 111 111 111 F F F
71202024-02-13T21:42:12.323Z 13 260-279 65 65 65 111 111 111 F F F
71212024-02-13T21:42:12.323Z 14 280-299 62 62 62 107 107 107 F F F
71222024-02-13T21:42:12.324Z 15 300-319 64 64 64 109 109 109 F F F
71232024-02-13T21:42:12.324Z 16 320-339 57 57 57 92 92 92 F F F
71242024-02-13T21:42:12.324Z 17 340-359 61 61 61 106 106 106 F F F
71252024-02-13T21:42:12.324Z 18 360-379 64 64 64 110 110 110 F F F
71262024-02-13T21:42:12.324Z 19 380-399 65 65 65 111 111 111 F F F
71272024-02-13T21:42:12.324Z 20 400-419 62 62 62 107 107 107 F F F
71282024-02-13T21:42:12.324Z 21 420-439 62 62 62 107 107 107 F F F
71292024-02-13T21:42:12.324Z 22 440-459 62 62 62 107 107 107 F F F
71302024-02-13T21:42:12.324Z 23 460-479 61 61 61 106 106 106 F F F
71312024-02-13T21:42:12.324Z 24 480-499 64 64 64 108 108 108 F F F
71322024-02-13T21:42:12.324Z 25 500-519 55 55 55 88 88 88 F F F
71332024-02-13T21:42:12.324Z 26 520-539 64 64 64 110 110 110 F F F
71342024-02-13T21:42:12.324Z 27 540-559 58 58 58 93 93 93 F F F
71352024-02-13T21:42:12.324Z 28 560-579 54 54 54 86 86 86 F F F
71362024-02-13T21:42:12.324Z 29 580-599 61 61 61 106 106 106 F F F
71372024-02-13T21:42:12.324ZMax gen: 65, Max flush: 111
71382024-02-13T21:42:12.324Z
71392024-02-13T21:42:12.324ZBegin loop 33
71402024-02-13T21:42:12.324ZDownstairs to restart: 1
71412024-02-13T21:42:12.337Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.125670047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
71422024-02-13T21:42:12.337Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
71432024-02-13T21:42:12.337Z,"time":"2024-02-13T21:42:11.125947826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
71442024-02-13T21:42:12.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.125994689Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
71452024-02-13T21:42:12.343Z{"msg":"Crucible f5b2cea7-57be-4649-9cb7-3bc0fb3a3718 has session id: b7e788d5-5c5c-4524-b139-05c5779abd4f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126025488Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71462024-02-13T21:42:12.343Z{"msg":"Upstairs opts: Upstairs UUID: f5b2cea7-57be-4649-9cb7-3bc0fb3a3718, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126087105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71472024-02-13T21:42:12.343Z{"msg":"Crucible stats registered with UUID: f5b2cea7-57be-4649-9cb7-3bc0fb3a3718","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126120533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71482024-02-13T21:42:12.343Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126135088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
71492024-02-13T21:42:12.344Z{"msg":"f5b2cea7-57be-4649-9cb7-3bc0fb3a3718 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126147154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71502024-02-13T21:42:12.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126167756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71512024-02-13T21:42:12.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126182561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71522024-02-13T21:42:12.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126196376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71532024-02-13T21:42:12.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126209851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71542024-02-13T21:42:12.344Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126223396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71552024-02-13T21:42:12.344Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126237151Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71562024-02-13T21:42:12.344Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126324239Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71572024-02-13T21:42:12.344Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126357737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71582024-02-13T21:42:12.344Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126372932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71592024-02-13T21:42:12.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126553346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71602024-02-13T21:42:12.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126801606Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71612024-02-13T21:42:12.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.126840052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"io task","client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71622024-02-13T21:42:12.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.127298506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71632024-02-13T21:42:12.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.127340261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71642024-02-13T21:42:12.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.127356535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71652024-02-13T21:42:12.344Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.1273705Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71662024-02-13T21:42:12.344Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.127910184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71672024-02-13T21:42:12.344Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.127952409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71682024-02-13T21:42:12.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128099575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71692024-02-13T21:42:12.344Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128145669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71702024-02-13T21:42:12.344Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128470551Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71712024-02-13T21:42:12.344Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128521372Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71722024-02-13T21:42:12.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128593126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71732024-02-13T21:42:12.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128631392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71742024-02-13T21:42:12.345Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128647886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71752024-02-13T21:42:12.345Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128668869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71762024-02-13T21:42:12.345Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.128821783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71772024-02-13T21:42:12.345Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129253727Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71782024-02-13T21:42:12.345Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129308617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71792024-02-13T21:42:12.345Z{"msg":"[0]R flush_numbers[0..12]: [107, 111, 111, 107, 109, 110, 107, 107, 107, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129344694Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71802024-02-13T21:42:12.345Z{"msg":"[0]R generation[0..12]: [62, 65, 65, 62, 64, 64, 62, 62, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129361118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71812024-02-13T21:42:12.345Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.12938313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71822024-02-13T21:42:12.345Z{"msg":"[1]R flush_numbers[0..12]: [107, 111, 111, 107, 109, 110, 107, 107, 107, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129398544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71832024-02-13T21:42:12.345Z{"msg":"[1]R generation[0..12]: [62, 65, 65, 62, 64, 64, 62, 62, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129412689Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71842024-02-13T21:42:12.345ZWait for a query_work_queue command to finish before sending IO
71852024-02-13T21:42:12.345Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.12943846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71862024-02-13T21:42:12.345Z{"msg":"[2]R flush_numbers[0..12]: [107, 111, 111, 107, 109, 110, 107, 107, 107, 111, 111, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129453514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71872024-02-13T21:42:12.345Z{"msg":"[2]R generation[0..12]: [62, 65, 65, 62, 64, 64, 62, 62, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129467959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71882024-02-13T21:42:12.345Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129482264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71892024-02-13T21:42:12.345Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129503986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71902024-02-13T21:42:12.345Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129518061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71912024-02-13T21:42:12.345Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129553868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71922024-02-13T21:42:12.345Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129567333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"":"downstairs","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71932024-02-13T21:42:12.351Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129588645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71942024-02-13T21:42:12.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129619164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"0","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71952024-02-13T21:42:12.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129636748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"1","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71962024-02-13T21:42:12.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129670256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"client":"2","":"downstairs","session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71972024-02-13T21:42:12.351Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129693487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71982024-02-13T21:42:12.351Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129727385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
71992024-02-13T21:42:12.351Z{"msg":"f5b2cea7-57be-4649-9cb7-3bc0fb3a3718 is now active with session: b7e788d5-5c5c-4524-b139-05c5779abd4f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129755545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
72002024-02-13T21:42:12.351Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129790812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402,"session_id":"b7e788d5-5c5c-4524-b139-05c5779abd4f"}
72012024-02-13T21:42:12.351Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:11.129827229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
72022024-02-13T21:42:12.351ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
72032024-02-13T21:42:12.351ZLoading write count information from file "/tmp/test_repair_verify.data"
72042024-02-13T21:42:12.351ZRun Repair workload
72052024-02-13T21:42:12.351Z01/30 Write block 219 len 12288 data: 9 6 6
72062024-02-13T21:42:12.351Z02/30 Read block 199 len 32768
72072024-02-13T21:42:12.351Z03/30 Write block 269 len 36864 data: 5 5 6 6 6 6 8 8 10
72082024-02-13T21:42:12.351Z04/30 Read block 175 len 4096
72092024-02-13T21:42:12.351Z05/30 Write block 285 len 8192 data: 3 3
72102024-02-13T21:42:12.351Z06/30 Write block 76 len 8192 data: 7 7
72112024-02-13T21:42:12.352Z07/30 Read block 176 len 4096
72122024-02-13T21:42:12.352Z08/30 Read block 28 len 28672
72132024-02-13T21:42:12.352Z09/30 Read block 141 len 16384
72142024-02-13T21:42:12.352Z10/30 Write block 502 len 28672 data: 3 2 4 4 6 6 6
72152024-02-13T21:42:12.352Z11/30 Read block 526 len 24576
72162024-02-13T21:42:12.355Z12/30 Read block 297 len 36864
72172024-02-13T21:42:12.355Z13/30 Write block 111 len 12288 data: 5 5 5
72182024-02-13T21:42:12.355Z14/30 Flush
72192024-02-13T21:42:12.358Z15/30 Write block 117 len 28672 data: 7 7 7 6 7 7 6
72202024-02-13T21:42:12.358Z16/30 Write block 2 len 32768 data: 5 5 6 7 6 6 5 6
72212024-02-13T21:42:12.358Z17/30 Read block 429 len 12288
72222024-02-13T21:42:12.359Z18/30 Read block 343 len 40960
72232024-02-13T21:42:12.361Z19/30 Write block 64 len 4096 data: 4
72242024-02-13T21:42:12.362Z20/30 Read block 320 len 16384
72252024-02-13T21:42:12.362Z21/30 Write block 331 len 24576 data: 6 8 8 8 10 10
72262024-02-13T21:42:12.362Z22/30 Read block 247 len 8192
72272024-02-13T21:42:12.365Z23/30 Read block 481 len 4096
72282024-02-13T21:42:12.365Z24/30 Write block 493 len 36864 data: 6 9 8 8 9 9 7 7 4
72292024-02-13T21:42:12.365Z25/30 Read block 325 len 20480
72302024-02-13T21:42:12.365Z26/30 Flush
72312024-02-13T21:42:12.368Z27/30 Write block 367 len 28672 data: 7 6 6 6 6 6 7
72322024-02-13T21:42:12.371Z28/30 Read block 83 len 8192
72332024-02-13T21:42:12.371Z29/30 Read block 465 len 40960
72342024-02-13T21:42:12.371Z30/30 Read block 310 len 12288
72352024-02-13T21:42:12.374Z----------------------------------------------------------------
72362024-02-13T21:42:12.374Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
72372024-02-13T21:42:12.374ZStates: Active Active Active
72382024-02-13T21:42:12.374ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72392024-02-13T21:42:12.374Z 1 Acked 1000 Write 3 Done Sent Done false
72402024-02-13T21:42:12.374Z 2 Acked 1001 Read 8 Done Sent Done false
72412024-02-13T21:42:12.374Z 3 Acked 1002 Write 9 Done Sent Done false
72422024-02-13T21:42:12.374Z 4 Acked 1003 Read 1 Done Sent Done false
72432024-02-13T21:42:12.374Z 5 Acked 1004 Write 2 Done Sent Done false
72442024-02-13T21:42:12.374Z 6 Acked 1005 Write 2 Done Sent Done false
72452024-02-13T21:42:12.374Z 7 Acked 1006 Read 1 Done Sent Done false
72462024-02-13T21:42:12.374Z 8 Acked 1007 Read 7 Done Sent Done false
72472024-02-13T21:42:12.374Z 9 Acked 1008 Read 4 Done Sent Done false
72482024-02-13T21:42:12.374Z 10 Acked 1009 Write 7 Done Sent Done false
72492024-02-13T21:42:12.374Z 11 Acked 1010 Read 6 Done Sent Done false
72502024-02-13T21:42:12.374Z 12 Acked 1011 Read 9 Done Sent Done false
72512024-02-13T21:42:12.374Z 13 Acked 1012 Write 3 Done Sent Done false
72522024-02-13T21:42:12.375Z 14 Acked 1013 Flush 0 Done Sent Done false
72532024-02-13T21:42:12.375Z 15 Acked 1014 Write 7 Done Sent Done false
72542024-02-13T21:42:12.375Z 16 Acked 1015 Write 8 Done Sent Done false
72552024-02-13T21:42:12.375Z 17 Acked 1016 Read 3 Done Sent Done false
72562024-02-13T21:42:12.375Z 18 Acked 1017 Read 10 Done Sent Done false
72572024-02-13T21:42:12.375Z 19 Acked 1018 Write 1 Done Sent Done false
72582024-02-13T21:42:12.375Z 20 Acked 1019 Read 4 Done Sent Done false
72592024-02-13T21:42:12.375Z 21 Acked 1020 Write 6 Done Sent Done false
72602024-02-13T21:42:12.375Z 22 Acked 1021 Read 2 Done Sent Done false
72612024-02-13T21:42:12.375Z 23 Acked 1022 Read 1 Done Sent Done false
72622024-02-13T21:42:12.375Z 24 Acked 1023 Write 9 Done Sent Done false
72632024-02-13T21:42:12.375Z 25 Acked 1024 Read 5 Done Sent Done false
72642024-02-13T21:42:12.375Z 26 Acked 1025 Flush 0 Done Sent Done false
72652024-02-13T21:42:12.375Z 27 Acked 1026 Write 7 Done Sent Done false
72662024-02-13T21:42:12.375Z 28 Acked 1027 Read 2 Done Sent Done false
72672024-02-13T21:42:12.375Z 29 Acked 1028 Read 10 Done Sent Done false
72682024-02-13T21:42:12.375Z 30 Acked 1029 Read 3 Done Sent Done false
72692024-02-13T21:42:12.375Z STATES DS:0 DS:1 DS:2 TOTAL
72702024-02-13T21:42:12.375Z New 0 0 0 0
72712024-02-13T21:42:12.375Z Sent 0 30 0 30
72722024-02-13T21:42:12.375Z Done 30 0 30 60
72732024-02-13T21:42:12.375Z Skipped 0 0 0 0
72742024-02-13T21:42:12.375Z Error 0 0 0 0
72752024-02-13T21:42:12.376ZLast Flush: 1025 0 1025
72762024-02-13T21:42:12.376ZDownstairs last five completed:
72772024-02-13T21:42:12.376ZUpstairs last five completed: 30 29 28 27 26
72782024-02-13T21:42:12.376Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:11.160392581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1402}
72792024-02-13T21:42:12.376ZWrote out file "/tmp/test_repair_verify.data"
72802024-02-13T21:42:12.379Z
72812024-02-13T21:42:12.379ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72822024-02-13T21:42:12.393ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72832024-02-13T21:42:12.393Z 0 000-019 66 62 66 113 107 113 F F F <---
72842024-02-13T21:42:12.393Z 1 020-039 65 65 65 111 111 111 F F F
72852024-02-13T21:42:12.393Z 2 040-059 65 65 65 111 111 111 F F F
72862024-02-13T21:42:12.393Z 3 060-079 66 62 66 113 107 113 F F F <---
72872024-02-13T21:42:12.393Z 4 080-099 64 64 64 109 109 109 F F F
72882024-02-13T21:42:12.393Z 5 100-119 66 64 66 113 110 113 F F F <---
72892024-02-13T21:42:12.393Z 6 120-139 66 62 66 113 107 113 F F F <---
72902024-02-13T21:42:12.393Z 7 140-159 62 62 62 107 107 107 F F F
72912024-02-13T21:42:12.393Z 8 160-179 62 62 62 107 107 107 F F F
72922024-02-13T21:42:12.397Z 9 180-199 65 65 65 111 111 111 F F F
72932024-02-13T21:42:12.397Z 10 200-219 66 65 66 112 111 112 F F F <---
72942024-02-13T21:42:12.397Z 11 220-239 66 64 66 112 110 112 F F F <---
72952024-02-13T21:42:12.397Z 12 240-259 65 65 65 111 111 111 F F F
72962024-02-13T21:42:12.397Z 13 260-279 66 65 66 112 111 112 F F F <---
72972024-02-13T21:42:12.397Z 14 280-299 66 62 66 112 107 112 F F F <---
72982024-02-13T21:42:12.397Z 15 300-319 64 64 64 109 109 109 F F F
72992024-02-13T21:42:12.397Z 16 320-339 66 57 66 113 92 113 F F F <---
73002024-02-13T21:42:12.397Z 17 340-359 61 61 61 106 106 106 F F F
73012024-02-13T21:42:12.397Z 18 360-379 64 64 64 110 110 110 T F T <---
73022024-02-13T21:42:12.397Z 19 380-399 65 65 65 111 111 111 F F F
73032024-02-13T21:42:12.397Z 20 400-419 62 62 62 107 107 107 F F F
73042024-02-13T21:42:12.397Z 21 420-439 62 62 62 107 107 107 F F F
73052024-02-13T21:42:12.397Z 22 440-459 62 62 62 107 107 107 F F F
73062024-02-13T21:42:12.397Z 23 460-479 61 61 61 106 106 106 F F F
73072024-02-13T21:42:12.397Z 24 480-499 66 64 66 113 108 113 F F F <---
73082024-02-13T21:42:12.397Z 25 500-519 66 55 66 113 88 113 F F F <---
73092024-02-13T21:42:12.397Z 26 520-539 64 64 64 110 110 110 F F F
73102024-02-13T21:42:12.397Z 27 540-559 58 58 58 93 93 93 F F F
73112024-02-13T21:42:12.397Z 28 560-579 54 54 54 86 86 86 F F F
73122024-02-13T21:42:12.397Z 29 580-599 61 61 61 106 106 106 F F F
73132024-02-13T21:42:12.397ZMax gen: 66, Max flush: 113
73142024-02-13T21:42:12.398ZError: Difference in extent metadata found!
73152024-02-13T21:42:12.398ZA Difference in extent metadata is expected here
73162024-02-13T21:42:12.398ZOn loop 33
73172024-02-13T21:42:12.398Z
73182024-02-13T21:42:12.401ZVerifying data now
73192024-02-13T21:42:16.994ZNo change after verify
73202024-02-13T21:42:16.995ZLoop: 33 Downstairs dump after verify (and repair):
73212024-02-13T21:42:17.011ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73222024-02-13T21:42:17.011Z 0 000-019 66 66 66 113 113 113 F F F
73232024-02-13T21:42:17.011Z 1 020-039 65 65 65 111 111 111 F F F
73242024-02-13T21:42:17.011Z 2 040-059 65 65 65 111 111 111 F F F
73252024-02-13T21:42:17.011Z 3 060-079 66 66 66 113 113 113 F F F
73262024-02-13T21:42:17.011Z 4 080-099 64 64 64 109 109 109 F F F
73272024-02-13T21:42:17.011Z 5 100-119 66 66 66 113 113 113 F F F
73282024-02-13T21:42:17.011Z 6 120-139 66 66 66 113 113 113 F F F
73292024-02-13T21:42:17.012Z 7 140-159 62 62 62 107 107 107 F F F
73302024-02-13T21:42:17.012Z 8 160-179 62 62 62 107 107 107 F F F
73312024-02-13T21:42:17.019Z 9 180-199 65 65 65 111 111 111 F F F
73322024-02-13T21:42:17.019Z 10 200-219 66 66 66 112 112 112 F F F
73332024-02-13T21:42:17.019Z 11 220-239 66 66 66 112 112 112 F F F
73342024-02-13T21:42:17.020Z 12 240-259 65 65 65 111 111 111 F F F
73352024-02-13T21:42:17.020Z 13 260-279 66 66 66 112 112 112 F F F
73362024-02-13T21:42:17.020Z 14 280-299 66 66 66 112 112 112 F F F
73372024-02-13T21:42:17.020Z 15 300-319 64 64 64 109 109 109 F F F
73382024-02-13T21:42:17.020Z 16 320-339 66 66 66 113 113 113 F F F
73392024-02-13T21:42:17.020Z 17 340-359 61 61 61 106 106 106 F F F
73402024-02-13T21:42:17.021Z 18 360-379 67 67 67 114 114 114 F F F
73412024-02-13T21:42:17.021Z 19 380-399 65 65 65 111 111 111 F F F
73422024-02-13T21:42:17.021Z 20 400-419 62 62 62 107 107 107 F F F
73432024-02-13T21:42:17.021Z 21 420-439 62 62 62 107 107 107 F F F
73442024-02-13T21:42:17.021Z 22 440-459 62 62 62 107 107 107 F F F
73452024-02-13T21:42:17.021Z 23 460-479 61 61 61 106 106 106 F F F
73462024-02-13T21:42:17.021Z 24 480-499 66 66 66 113 113 113 F F F
73472024-02-13T21:42:17.021Z 25 500-519 66 66 66 113 113 113 F F F
73482024-02-13T21:42:17.022Z 26 520-539 64 64 64 110 110 110 F F F
73492024-02-13T21:42:17.022Z 27 540-559 58 58 58 93 93 93 F F F
73502024-02-13T21:42:17.022Z 28 560-579 54 54 54 86 86 86 F F F
73512024-02-13T21:42:17.022Z 29 580-599 61 61 61 106 106 106 F F F
73522024-02-13T21:42:17.022ZMax gen: 67, Max flush: 114
73532024-02-13T21:42:17.022Z
73542024-02-13T21:42:17.022ZBegin loop 34
73552024-02-13T21:42:17.022ZDownstairs to restart: 2
73562024-02-13T21:42:17.029Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815374741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
73572024-02-13T21:42:17.029Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815633417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
73582024-02-13T21:42:17.029Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815652221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
73592024-02-13T21:42:17.037Z{"msg":"Crucible ad23055c-3013-4248-91b6-bc4f1fe30192 has session id: 1f08b914-fae3-475a-9eba-fff857923c26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815683819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73602024-02-13T21:42:17.037Z{"msg":"Upstairs opts: Upstairs UUID: ad23055c-3013-4248-91b6-bc4f1fe30192, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
73612024-02-13T21:42:17.037Z,"time":"2024-02-13T21:42:15.815721575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73622024-02-13T21:42:17.037Z{"msg":"Crucible stats registered with UUID: ad23055c-3013-4248-91b6-bc4f1fe30192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815754443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73632024-02-13T21:42:17.037Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815797258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
73642024-02-13T21:42:17.037Z{"msg":"ad23055c-3013-4248-91b6-bc4f1fe30192 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815910777Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73652024-02-13T21:42:17.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815942285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73662024-02-13T21:42:17.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.81595779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73672024-02-13T21:42:17.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.815979982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73682024-02-13T21:42:17.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816009551Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73692024-02-13T21:42:17.038Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816031773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73702024-02-13T21:42:17.038Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816049316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73712024-02-13T21:42:17.038Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816073657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73722024-02-13T21:42:17.038Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816114833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73732024-02-13T21:42:17.038Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816139904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73742024-02-13T21:42:17.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.81617691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73752024-02-13T21:42:17.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.81620424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73762024-02-13T21:42:17.038Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816229681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"io task","client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73772024-02-13T21:42:17.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816895769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73782024-02-13T21:42:17.038Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816938084Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73792024-02-13T21:42:17.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.816966644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73802024-02-13T21:42:17.038Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817023143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73812024-02-13T21:42:17.038Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817522212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73822024-02-13T21:42:17.038Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817681274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73832024-02-13T21:42:17.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817923197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73842024-02-13T21:42:17.038Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817959174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73852024-02-13T21:42:17.038Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.817984515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73862024-02-13T21:42:17.038Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818167828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73872024-02-13T21:42:17.038Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818210832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73882024-02-13T21:42:17.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818227047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73892024-02-13T21:42:17.038Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818248099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73902024-02-13T21:42:17.039Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818262294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73912024-02-13T21:42:17.039Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.818726865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73922024-02-13T21:42:17.039Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.8191531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73932024-02-13T21:42:17.039Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819189867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73942024-02-13T21:42:17.039Z{"msg":"[0]R flush_numbers[0..12]: [113, 111, 111, 113, 109, 113, 113, 107, 107, 111, 112, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819206831Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73952024-02-13T21:42:17.039Z{"msg":"[0]R generation[0..12]: [66, 65, 65, 66, 64, 66, 66, 62, 62, 65, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819228453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73962024-02-13T21:42:17.039Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819243418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73972024-02-13T21:42:17.039Z{"msg":"[1]R flush_numbers[0..12]: [113, 111, 111, 113, 109, 113, 113, 107, 107, 111, 112, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.81926414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73982024-02-13T21:42:17.039Z{"msg":"[1]R generation[0..12]: [66, 65, 65, 66, 64, 66, 66, 62, 62, 65, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819277975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
73992024-02-13T21:42:17.039Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819298278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74002024-02-13T21:42:17.039Z{"msg":"[2]R flush_numbers[0..12]: [113, 111, 111, 113, 109, 113, 113, 107, 107, 111, 112, 112]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
74012024-02-13T21:42:17.039Z,"time":"2024-02-13T21:42:15.819312643Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74022024-02-13T21:42:17.039Z{"msg":"[2]R generation[0..12]: [66, 65, 65, 66, 64, 66, 66, 62, 62, 65, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819336824Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74032024-02-13T21:42:17.039Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819357496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74042024-02-13T21:42:17.040Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819370901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74052024-02-13T21:42:17.040Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819391294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74062024-02-13T21:42:17.040Z{"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819405169Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74072024-02-13T21:42:17.040Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819418994Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"":"downstairs","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74082024-02-13T21:42:17.040Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819438957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74092024-02-13T21:42:17.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819451123Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"0","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74102024-02-13T21:42:17.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819470575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"1","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74112024-02-13T21:42:17.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.8194842Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"client":"2","":"downstairs","session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74122024-02-13T21:42:17.040Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819503753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74132024-02-13T21:42:17.040Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819516379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74142024-02-13T21:42:17.041Z{"msg":"ad23055c-3013-4248-91b6-bc4f1fe30192 is now active with session: 1f08b914-fae3-475a-9eba-fff857923c26","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819530674Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74152024-02-13T21:42:17.041Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819550207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410,"session_id":"1f08b914-fae3-475a-9eba-fff857923c26"}
74162024-02-13T21:42:17.041Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:15.819563152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
74172024-02-13T21:42:17.041ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
74182024-02-13T21:42:17.046ZLoading write count information from file "/tmp/test_repair_verify.data"
74192024-02-13T21:42:17.046ZRun Repair workload
74202024-02-13T21:42:17.046Z01/30 Write block 44 len 36864 data: 4 4 5 4 5 7 5 4 4
74212024-02-13T21:42:17.046Z02/30 Write block 222 len 16384 data: 7 7 7 6
74222024-02-13T21:42:17.046Z03/30 Flush
74232024-02-13T21:42:17.046Z04/30 Write block 482 len 24576 data: 6 5 4 4 5 5
74242024-02-13T21:42:17.046Z05/30 Write block 20 len 24576 data: 6 6 7 4 4 3
74252024-02-13T21:42:17.046Z06/30 Write block 388 len 40960 data: 6 5 5 5 5 6 5 3 3 2
74262024-02-13T21:42:17.046Z07/30 Write block 237 len 20480 data: 6 9 9 9 8
74272024-02-13T21:42:17.046Z08/30 Read block 382 len 8192
74282024-02-13T21:42:17.046Z09/30 Write block 125 len 20480 data: 8 8 7 6 6
74292024-02-13T21:42:17.046Z10/30 Read block 68 len 4096
74302024-02-13T21:42:17.047Z11/30 Write block 460 len 32768 data: 8 7 7 6 8 7 6 7
74312024-02-13T21:42:17.047Z12/30 Read block 322 len 8192
74322024-02-13T21:42:17.051Z13/30 Read block 487 len 40960
74332024-02-13T21:42:17.051Z14/30 Read block 553 len 4096
74342024-02-13T21:42:17.051Z15/30 Write block 150 len 40960 data: 3 4 4 5 5 6 6 7 8 6
74352024-02-13T21:42:17.051Z16/30 Read block 145 len 36864
74362024-02-13T21:42:17.051Z17/30 Read block 331 len 28672
74372024-02-13T21:42:17.055Z18/30 Write block 93 len 24576 data: 3 3 5 5 6 7
74382024-02-13T21:42:17.055Z19/30 Read block 215 len 4096
74392024-02-13T21:42:17.055Z20/30 Write block 326 len 40960 data: 7 7 4 3 3 7 9 9 9 11
74402024-02-13T21:42:17.055Z21/30 Write block 319 len 28672 data: 6 6 6 5 7 6 7
74412024-02-13T21:42:17.056Z22/30 Read block 487 len 12288
74422024-02-13T21:42:17.065Z23/30 Read block 508 len 20480
74432024-02-13T21:42:17.065Z24/30 Read block 4 len 24576
74442024-02-13T21:42:17.065Z25/30 Flush
74452024-02-13T21:42:17.065Z26/30 Write block 524 len 4096 data: 6
74462024-02-13T21:42:17.065Z27/30 Read block 439 len 12288
74472024-02-13T21:42:17.065Z28/30 Read block 168 len 28672
74482024-02-13T21:42:17.065Z29/30 Write block 9 len 24576 data: 7 7 9 8 8 9
74492024-02-13T21:42:17.065Z30/30 Write block 194 len 36864 data: 9 8 9 10 11 11 8 7 7
74502024-02-13T21:42:17.069Z----------------------------------------------------------------
74512024-02-13T21:42:17.069Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74522024-02-13T21:42:17.069ZStates: Active Active Active
74532024-02-13T21:42:17.069ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74542024-02-13T21:42:17.069Z 1 Acked 1000 Write 9 Done Done Sent false
74552024-02-13T21:42:17.069Z 2 Acked 1001 Write 4 Done Done Sent false
74562024-02-13T21:42:17.069Z 3 Acked 1002 Flush 0 Done Done Sent false
74572024-02-13T21:42:17.069Z 4 Acked 1003 Write 6 Done Done Sent false
74582024-02-13T21:42:17.069Z 5 Acked 1004 Write 6 Done Done Sent false
74592024-02-13T21:42:17.069Z 6 Acked 1005 Write 10 Done Done Sent false
74602024-02-13T21:42:17.069Z 7 Acked 1006 Write 5 Done Done Sent false
74612024-02-13T21:42:17.069Z 8 Acked 1007 Read 2 Done Done Sent false
74622024-02-13T21:42:17.069Z 9 Acked 1008 Write 5 Done Done Sent false
74632024-02-13T21:42:17.069Z 10 Acked 1009 Read 1 Done Done Sent false
74642024-02-13T21:42:17.069Z 11 Acked 1010 Write 8 Done Done Sent false
74652024-02-13T21:42:17.069Z 12 Acked 1011 Read 2 Done Done Sent false
74662024-02-13T21:42:17.069Z 13 Acked 1012 Read 10 Done Done Sent false
74672024-02-13T21:42:17.069Z 14 Acked 1013 Read 1 Done Done Sent false
74682024-02-13T21:42:17.070Z 15 Acked 1014 Write 10 Done Done Sent false
74692024-02-13T21:42:17.070Z 16 Acked 1015 Read 9 Done Done Sent false
74702024-02-13T21:42:17.070Z 17 Acked 1016 Read 7 Done Done Sent false
74712024-02-13T21:42:17.070Z 18 Acked 1017 Write 6 Done Done Sent false
74722024-02-13T21:42:17.070Z 19 Acked 1018 Read 1 Done Done Sent false
74732024-02-13T21:42:17.070Z 20 Acked 1019 Write 10 Done Done Sent false
74742024-02-13T21:42:17.070Z 21 Acked 1020 Write 7 Done Done Sent false
74752024-02-13T21:42:17.070Z 22 Acked 1021 Read 3 Done Done Sent false
74762024-02-13T21:42:17.070Z 23 Acked 1022 Read 5 Done Done Sent false
74772024-02-13T21:42:17.070Z 24 Acked 1023 Read 6 Done Done Sent false
74782024-02-13T21:42:17.070Z 25 Acked 1024 Flush 0 Done Done Sent false
74792024-02-13T21:42:17.070Z 26 Acked 1025 Write 1 Done Done Sent false
74802024-02-13T21:42:17.070Z 27 Acked 1026 Read 3 Done Done Sent false
74812024-02-13T21:42:17.070Z 28 Acked 1027 Read 7 Done Done Sent false
74822024-02-13T21:42:17.070Z 29 Acked 1028 Write 6 Done Done Sent false
74832024-02-13T21:42:17.070Z 30 Acked 1029 Write 9 Sent Sent Sent false
74842024-02-13T21:42:17.070Z STATES DS:0 DS:1 DS:2 TOTAL
74852024-02-13T21:42:17.070Z New 0 0 0 0
74862024-02-13T21:42:17.070Z Sent 1 1 30 32
74872024-02-13T21:42:17.070Z Done 29 29 0 58
74882024-02-13T21:42:17.070Z Skipped 0 0 0 0
74892024-02-13T21:42:17.070Z Error 0 0 0 0
74902024-02-13T21:42:17.070ZLast Flush: 1024 1024 0
74912024-02-13T21:42:17.070ZDownstairs last five completed:
74922024-02-13T21:42:17.070ZUpstairs last five completed: 30 29 28 27 26
74932024-02-13T21:42:17.071Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:15.852878757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1410}
74942024-02-13T21:42:17.071ZWrote out file "/tmp/test_repair_verify.data"
74952024-02-13T21:42:17.074Z
74962024-02-13T21:42:17.074ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74972024-02-13T21:42:17.088ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74982024-02-13T21:42:17.089Z 0 000-019 66 66 66 113 113 113 T T F <---
74992024-02-13T21:42:17.089Z 1 020-039 68 68 65 116 116 111 F F F <---
75002024-02-13T21:42:17.089Z 2 040-059 68 68 65 115 115 111 F F F <---
75012024-02-13T21:42:17.089Z 3 060-079 66 66 66 113 113 113 F F F
75022024-02-13T21:42:17.089Z 4 080-099 68 68 64 116 116 109 F F F <---
75032024-02-13T21:42:17.089Z 5 100-119 66 66 66 113 113 113 F F F
75042024-02-13T21:42:17.089Z 6 120-139 68 68 66 116 116 113 F F F <---
75052024-02-13T21:42:17.089Z 7 140-159 68 68 62 116 116 107 F F F <---
75062024-02-13T21:42:17.089Z 8 160-179 62 62 62 107 107 107 F F F
75072024-02-13T21:42:17.089Z 9 180-199 65 65 65 111 111 111 T T F <---
75082024-02-13T21:42:17.092Z 10 200-219 66 66 66 112 112 112 T T F <---
75092024-02-13T21:42:17.092Z 11 220-239 68 68 66 116 116 112 F F F <---
75102024-02-13T21:42:17.092Z 12 240-259 68 68 65 116 116 111 F F F <---
75112024-02-13T21:42:17.092Z 13 260-279 66 66 66 112 112 112 F F F
75122024-02-13T21:42:17.092Z 14 280-299 66 66 66 112 112 112 F F F
75132024-02-13T21:42:17.092Z 15 300-319 68 68 64 116 116 109 F F F <---
75142024-02-13T21:42:17.093Z 16 320-339 68 68 66 116 116 113 F F F <---
75152024-02-13T21:42:17.093Z 17 340-359 61 61 61 106 106 106 F F F
75162024-02-13T21:42:17.093Z 18 360-379 67 67 67 114 114 114 F F F
75172024-02-13T21:42:17.093Z 19 380-399 68 68 65 116 116 111 F F F <---
75182024-02-13T21:42:17.093Z 20 400-419 62 62 62 107 107 107 F F F
75192024-02-13T21:42:17.093Z 21 420-439 62 62 62 107 107 107 F F F
75202024-02-13T21:42:17.093Z 22 440-459 62 62 62 107 107 107 F F F
75212024-02-13T21:42:17.093Z 23 460-479 68 68 61 116 116 106 F F F <---
75222024-02-13T21:42:17.093Z 24 480-499 68 68 66 116 116 113 F F F <---
75232024-02-13T21:42:17.093Z 25 500-519 66 66 66 113 113 113 F F F
75242024-02-13T21:42:17.093Z 26 520-539 64 64 64 110 110 110 T T F <---
75252024-02-13T21:42:17.093Z 27 540-559 58 58 58 93 93 93 F F F
75262024-02-13T21:42:17.093Z 28 560-579 54 54 54 86 86 86 F F F
75272024-02-13T21:42:17.093Z 29 580-599 61 61 61 106 106 106 F F F
75282024-02-13T21:42:17.093ZMax gen: 68, Max flush: 116
75292024-02-13T21:42:17.093ZError: Difference in extent metadata found!
75302024-02-13T21:42:17.093ZA Difference in extent metadata is expected here
75312024-02-13T21:42:17.093ZOn loop 34
75322024-02-13T21:42:17.093Z
75332024-02-13T21:42:17.096ZVerifying data now
75342024-02-13T21:42:21.852ZNo change after verify
75352024-02-13T21:42:21.859ZLoop: 34 Downstairs dump after verify (and repair):
75362024-02-13T21:42:21.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75372024-02-13T21:42:21.874Z 0 000-019 69 69 69 117 117 117 F F F
75382024-02-13T21:42:21.874Z 1 020-039 68 68 68 116 116 116 F F F
75392024-02-13T21:42:21.874Z 2 040-059 68 68 68 115 115 115 F F F
75402024-02-13T21:42:21.875Z 3 060-079 66 66 66 113 113 113 F F F
75412024-02-13T21:42:21.875Z 4 080-099 68 68 68 116 116 116 F F F
75422024-02-13T21:42:21.875Z 5 100-119 66 66 66 113 113 113 F F F
75432024-02-13T21:42:21.875Z 6 120-139 68 68 68 116 116 116 F F F
75442024-02-13T21:42:21.875Z 7 140-159 68 68 68 116 116 116 F F F
75452024-02-13T21:42:21.876Z 8 160-179 62 62 62 107 107 107 F F F
75462024-02-13T21:42:21.876Z 9 180-199 69 69 69 117 117 117 F F F
75472024-02-13T21:42:21.876Z 10 200-219 69 69 69 117 117 117 F F F
75482024-02-13T21:42:21.876Z 11 220-239 68 68 68 116 116 116 F F F
75492024-02-13T21:42:21.877Z 12 240-259 68 68 68 116 116 116 F F F
75502024-02-13T21:42:21.877Z 13 260-279 66 66 66 112 112 112 F F F
75512024-02-13T21:42:21.877Z 14 280-299 66 66 66 112 112 112 F F F
75522024-02-13T21:42:21.877Z 15 300-319 68 68 68 116 116 116 F F F
75532024-02-13T21:42:21.877Z 16 320-339 68 68 68 116 116 116 F F F
75542024-02-13T21:42:21.878Z 17 340-359 61 61 61 106 106 106 F F F
75552024-02-13T21:42:21.878Z 18 360-379 67 67 67 114 114 114 F F F
75562024-02-13T21:42:21.878Z 19 380-399 68 68 68 116 116 116 F F F
75572024-02-13T21:42:21.878Z 20 400-419 62 62 62 107 107 107 F F F
75582024-02-13T21:42:21.878Z 21 420-439 62 62 62 107 107 107 F F F
75592024-02-13T21:42:21.879Z 22 440-459 62 62 62 107 107 107 F F F
75602024-02-13T21:42:21.879Z 23 460-479 68 68 68 116 116 116 F F F
75612024-02-13T21:42:21.879Z 24 480-499 68 68 68 116 116 116 F F F
75622024-02-13T21:42:21.879Z 25 500-519 66 66 66 113 113 113 F F F
75632024-02-13T21:42:21.879Z 26 520-539 69 69 69 117 117 117 F F F
75642024-02-13T21:42:21.880Z 27 540-559 58 58 58 93 93 93 F F F
75652024-02-13T21:42:21.880Z 28 560-579 54 54 54 86 86 86 F F F
75662024-02-13T21:42:21.880Z 29 580-599 61 61 61 106 106 106 F F F
75672024-02-13T21:42:21.880ZMax gen: 69, Max flush: 117
75682024-02-13T21:42:21.880Z
75692024-02-13T21:42:21.881ZBegin loop 35
75702024-02-13T21:42:21.881ZDownstairs to restart: 0
75712024-02-13T21:42:21.912Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.670839329Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
75722024-02-13T21:42:21.912Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671091957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
75732024-02-13T21:42:21.912Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671127334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
75742024-02-13T21:42:21.912Z{"msg":"Crucible 2940dc38-ec0a-46c6-a9eb-050ff94b6d6f has session id: ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671151296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"Crucible runtime is spawned
75752024-02-13T21:42:21.912Zsession_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75762024-02-13T21:42:21.912Z{"msg":"Upstairs opts: Upstairs UUID: 2940dc38-ec0a-46c6-a9eb-050ff94b6d6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671205866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75772024-02-13T21:42:21.912Z{"msg":"Crucible stats registered with UUID: 2940dc38-ec0a-46c6-a9eb-050ff94b6d6f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671232006Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75782024-02-13T21:42:21.913Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671272462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
75792024-02-13T21:42:21.913Z{"msg":"2940dc38-ec0a-46c6-a9eb-050ff94b6d6f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671377134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75802024-02-13T21:42:21.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67141504Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75812024-02-13T21:42:21.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67144232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75822024-02-13T21:42:21.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671466361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75832024-02-13T21:42:21.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671488653Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75842024-02-13T21:42:21.946Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671529009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75852024-02-13T21:42:21.946Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671585378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75862024-02-13T21:42:21.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671656872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75872024-02-13T21:42:21.946Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67171945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75882024-02-13T21:42:21.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67174429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75892024-02-13T21:42:21.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671777259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75902024-02-13T21:42:21.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671802619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75912024-02-13T21:42:21.947Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.671836157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"io task","client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75922024-02-13T21:42:21.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.672326119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75932024-02-13T21:42:21.947Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.672377271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75942024-02-13T21:42:21.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.672400922Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75952024-02-13T21:42:21.947Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67243514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75962024-02-13T21:42:21.947Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.672964508Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75972024-02-13T21:42:21.947Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67306961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75982024-02-13T21:42:21.947Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673386575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
75992024-02-13T21:42:21.948Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67342732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76002024-02-13T21:42:21.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673453721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76012024-02-13T21:42:21.948Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673485989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76022024-02-13T21:42:21.948Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67351057Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76032024-02-13T21:42:21.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673541379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76042024-02-13T21:42:21.948Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673573987Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76052024-02-13T21:42:21.948Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673606155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76062024-02-13T21:42:21.948Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.673836552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76072024-02-13T21:42:21.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674250232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76082024-02-13T21:42:21.948Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674297195Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76092024-02-13T21:42:21.949Z{"msg":"[0]R flush_numbers[0..12]: [117, 116, 115, 113, 116, 113, 116, 116, 107, 117, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674315798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76102024-02-13T21:42:21.949Z{"msg":"[0]R generation[0..12]: [69, 68, 68, 66, 68, 66, 68, 68, 62, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.67433774Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76112024-02-13T21:42:21.949Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674353064Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76122024-02-13T21:42:21.949Z{"msg":"[1]R flush_numbers[0..12]: [117, 116, 115, 113, 116, 113, 116, 116, 107, 117, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674373667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76132024-02-13T21:42:21.949Z{"msg":"[1]R generation[0..12]: [69, 68, 68, 66, 68, 66, 68, 68, 62, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674388022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76142024-02-13T21:42:21.949Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674408694Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76152024-02-13T21:42:21.949Z{"msg":"[2]R flush_numbers[0..12]: [117, 116, 115, 113, 116, 113, 116, 116, 107, 117, 117, 116]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
76162024-02-13T21:42:21.949Z,"time":"2024-02-13T21:42:20.674422989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76172024-02-13T21:42:21.949Z{"msg":"[2]R generation[0..12]: [69, 68, 68, 66, 68, 66, 68, 68, 62, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674450179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76182024-02-13T21:42:21.949Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674471181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76192024-02-13T21:42:21.949Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674484856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76202024-02-13T21:42:21.949Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674505369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76212024-02-13T21:42:21.949Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674518644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76222024-02-13T21:42:21.949Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674538177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"":"downstairs","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76232024-02-13T21:42:21.949Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674552052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76242024-02-13T21:42:21.949Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674570296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"0","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76252024-02-13T21:42:21.949Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674583711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"1","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76262024-02-13T21:42:21.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674597506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"client":"2","":"downstairs","session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76272024-02-13T21:42:21.950Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674617308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76282024-02-13T21:42:21.950Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674630004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76292024-02-13T21:42:21.950Z{"msg":"2940dc38-ec0a-46c6-a9eb-050ff94b6d6f is now active with session: ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674649277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76302024-02-13T21:42:21.950Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674661452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418,"session_id":"ddf47b5b-0969-4e1f-ac9d-fb4a7db5b81a"}
76312024-02-13T21:42:21.950Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:20.674680656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
76322024-02-13T21:42:21.950ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76332024-02-13T21:42:21.950ZLoading write count information from file "/tmp/test_repair_verify.data"
76342024-02-13T21:42:21.950ZRun Repair workload
76352024-02-13T21:42:21.950Z01/30 Write block 303 len 40960 data: 5 5 5 4 7 6 5 6 8 10
76362024-02-13T21:42:21.950Z02/30 Write block 132 len 12288 data: 8 8 6
76372024-02-13T21:42:21.950Z03/30 Write block 51 len 8192 data: 5 5
76382024-02-13T21:42:21.950Z04/30 Read block 182 len 36864
76392024-02-13T21:42:21.950Z05/30 Write block 426 len 4096 data: 7
76402024-02-13T21:42:21.950Z06/30 Write block 15 len 40960 data: 10 12 11 11 8 7 7 8 5 5
76412024-02-13T21:42:21.950Z07/30 Flush
76422024-02-13T21:42:21.951Z08/30 Write block 153 len 12288 data: 6 6 7
76432024-02-13T21:42:21.951Z09/30 Read block 201 len 12288
76442024-02-13T21:42:21.951Z10/30 Write block 375 len 12288 data: 9 6 6
76452024-02-13T21:42:21.951Z11/30 Read block 201 len 24576
76462024-02-13T21:42:21.951Z12/30 Read block 355 len 12288
76472024-02-13T21:42:21.951Z13/30 Write block 260 len 12288 data: 3 3 3
76482024-02-13T21:42:21.951Z14/30 Read block 466 len 4096
76492024-02-13T21:42:21.951Z15/30 Read block 191 len 8192
76502024-02-13T21:42:21.951Z16/30 Read block 230 len 4096
76512024-02-13T21:42:21.951Z17/30 Write block 502 len 20480 data: 4 3 5 5 7
76522024-02-13T21:42:21.951Z18/30 Read block 32 len 20480
76532024-02-13T21:42:21.951Z19/30 Write block 371 len 8192 data: 7 7
76542024-02-13T21:42:21.952Z20/30 Read block 305 len 20480
76552024-02-13T21:42:21.952Z21/30 Read block 313 len 28672
76562024-02-13T21:42:21.952Z22/30 Flush
76572024-02-13T21:42:21.952Z23/30 Write block 10 len 8192 data: 8 10
76582024-02-13T21:42:21.952Z24/30 Read block 412 len 4096
76592024-02-13T21:42:21.952Z25/30 Read block 480 len 24576
76602024-02-13T21:42:21.952Z26/30 Write block 189 len 36864 data: 9 9 8 7 7 10 9 10 11
76612024-02-13T21:42:21.952Z27/30 Read block 67 len 36864
76622024-02-13T21:42:21.952Z28/30 Read block 51 len 36864
76632024-02-13T21:42:21.952Z29/30 Read block 539 len 40960
76642024-02-13T21:42:21.952Z30/30 Read block 531 len 28672
76652024-02-13T21:42:21.953Z----------------------------------------------------------------
76662024-02-13T21:42:21.953Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76672024-02-13T21:42:21.953ZStates: Active Active Active
76682024-02-13T21:42:21.953ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76692024-02-13T21:42:21.953Z 1 Acked 1000 Write 10 Done Done Done false
76702024-02-13T21:42:21.953Z 2 Acked 1001 Write 3 Done Done Done false
76712024-02-13T21:42:21.953Z 3 Acked 1002 Write 2 Done Done Done false
76722024-02-13T21:42:21.953Z 4 Acked 1003 Read 9 Sent Done Done false
76732024-02-13T21:42:21.953Z 5 Acked 1004 Write 1 Sent Done Done false
76742024-02-13T21:42:21.953Z 6 Acked 1005 Write 10 Sent Done Done false
76752024-02-13T21:42:21.953Z 7 Acked 1006 Flush 0 Sent Done Done false
76762024-02-13T21:42:21.953Z 8 Acked 1007 Write 3 Sent Done Done false
76772024-02-13T21:42:21.953Z 9 Acked 1008 Read 3 Sent Done Done false
76782024-02-13T21:42:21.953Z 10 Acked 1009 Write 3 Sent Done Done false
76792024-02-13T21:42:21.953Z 11 Acked 1010 Read 6 Sent Done Done false
76802024-02-13T21:42:21.954Z 12 Acked 1011 Read 3 Sent Done Done false
76812024-02-13T21:42:21.954Z 13 Acked 1012 Write 3 Sent Done Done false
76822024-02-13T21:42:21.954Z 14 Acked 1013 Read 1 Sent Done Done false
76832024-02-13T21:42:21.960Z 15 Acked 1014 Read 2 Sent Done Done false
76842024-02-13T21:42:21.960Z 16 Acked 1015 Read 1 Sent Done Done false
76852024-02-13T21:42:21.960Z 17 Acked 1016 Write 5 Sent Done Done false
76862024-02-13T21:42:21.960Z 18 Acked 1017 Read 5 Sent Done Done false
76872024-02-13T21:42:21.961Z 19 Acked 1018 Write 2 Sent Done Done false
76882024-02-13T21:42:21.961Z 20 Acked 1019 Read 5 Sent Done Done false
76892024-02-13T21:42:21.961Z 21 Acked 1020 Read 7 Sent Done Done false
76902024-02-13T21:42:21.961Z 22 Acked 1021 Flush 0 Sent Done Done false
76912024-02-13T21:42:21.961Z 23 Acked 1022 Write 2 Sent Done Done false
76922024-02-13T21:42:21.961Z 24 Acked 1023 Read 1 Sent Done Done false
76932024-02-13T21:42:21.961Z 25 Acked 1024 Read 6 Sent Done Done false
76942024-02-13T21:42:21.961Z 26 Acked 1025 Write 9 Sent Done Done false
76952024-02-13T21:42:21.961Z 27 Acked 1026 Read 9 Sent Done Done false
76962024-02-13T21:42:21.961Z 28 Acked 1027 Read 9 Sent Done Done false
76972024-02-13T21:42:21.961Z 29 Acked 1028 Read 10 Sent Done Done false
76982024-02-13T21:42:21.961Z 30 Acked 1029 Read 7 Sent Done Sent false
76992024-02-13T21:42:21.961Z STATES DS:0 DS:1 DS:2 TOTAL
77002024-02-13T21:42:21.961Z New 0 0 0 0
77012024-02-13T21:42:21.961Z Sent 27 0 1 28
77022024-02-13T21:42:21.961Z Done 3 30 29 62
77032024-02-13T21:42:21.961Z Skipped 0 0 0 0
77042024-02-13T21:42:21.961Z Error 0 0 0 0
77052024-02-13T21:42:21.961ZLast Flush: 0 1021 1021
77062024-02-13T21:42:21.962ZDownstairs last five completed:
77072024-02-13T21:42:21.962ZUpstairs last five completed: 30 29 28 27 26
77082024-02-13T21:42:21.962Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:20.704838386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1418}
77092024-02-13T21:42:21.962ZWrote out file "/tmp/test_repair_verify.data"
77102024-02-13T21:42:21.962Z
77112024-02-13T21:42:21.962ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
77122024-02-13T21:42:21.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77132024-02-13T21:42:21.962Z 0 000-019 69 70 70 117 118 118 F T T <---
77142024-02-13T21:42:21.962Z 1 020-039 68 70 70 116 118 118 F F F <---
77152024-02-13T21:42:21.962Z 2 040-059 68 70 70 115 118 118 T F F <---
77162024-02-13T21:42:21.962Z 3 060-079 66 66 66 113 113 113 F F F
77172024-02-13T21:42:21.962Z 4 080-099 68 68 68 116 116 116 F F F
77182024-02-13T21:42:21.962Z 5 100-119 66 66 66 113 113 113 F F F
77192024-02-13T21:42:21.962Z 6 120-139 68 70 70 116 118 118 T F F <---
77202024-02-13T21:42:21.962Z 7 140-159 68 70 70 116 119 119 F F F <---
77212024-02-13T21:42:21.962Z 8 160-179 62 62 62 107 107 107 F F F
77222024-02-13T21:42:21.962Z 9 180-199 69 69 69 117 117 117 F T T <---
77232024-02-13T21:42:21.962Z 10 200-219 69 69 69 117 117 117 F F F
77242024-02-13T21:42:21.962Z 11 220-239 68 68 68 116 116 116 F F F
77252024-02-13T21:42:21.962Z 12 240-259 68 68 68 116 116 116 F F F
77262024-02-13T21:42:21.963Z 13 260-279 66 70 70 112 119 119 F F F <---
77272024-02-13T21:42:21.963Z 14 280-299 66 66 66 112 112 112 F F F
77282024-02-13T21:42:21.963Z 15 300-319 68 70 70 116 118 118 T F F <---
77292024-02-13T21:42:21.963Z 16 320-339 68 68 68 116 116 116 F F F
77302024-02-13T21:42:21.963Z 17 340-359 61 61 61 106 106 106 F F F
77312024-02-13T21:42:21.963Z 18 360-379 67 70 70 114 119 119 F F F <---
77322024-02-13T21:42:21.963Z 19 380-399 68 68 68 116 116 116 F F F
77332024-02-13T21:42:21.963Z 20 400-419 62 62 62 107 107 107 F F F
77342024-02-13T21:42:21.963Z 21 420-439 62 70 70 107 118 118 F F F <---
77352024-02-13T21:42:21.963Z 22 440-459 62 62 62 107 107 107 F F F
77362024-02-13T21:42:21.963Z 23 460-479 68 68 68 116 116 116 F F F
77372024-02-13T21:42:21.963Z 24 480-499 68 68 68 116 116 116 F F F
77382024-02-13T21:42:21.963Z 25 500-519 66 70 70 113 119 119 F F F <---
77392024-02-13T21:42:21.963Z 26 520-539 69 69 69 117 117 117 F F F
77402024-02-13T21:42:21.963Z 27 540-559 58 58 58 93 93 93 F F F
77412024-02-13T21:42:21.963Z 28 560-579 54 54 54 86 86 86 F F F
77422024-02-13T21:42:21.963Z 29 580-599 61 61 61 106 106 106 F F F
77432024-02-13T21:42:21.963ZMax gen: 70, Max flush: 119
77442024-02-13T21:42:21.964ZError: Difference in extent metadata found!
77452024-02-13T21:42:21.964ZA Difference in extent metadata is expected here
77462024-02-13T21:42:21.964ZOn loop 35
77472024-02-13T21:42:21.964Z
77482024-02-13T21:42:21.964ZVerifying data now
77492024-02-13T21:42:27.511ZNo change after verify
77502024-02-13T21:42:27.512ZLoop: 35 Downstairs dump after verify (and repair):
77512024-02-13T21:42:27.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77522024-02-13T21:42:27.540Z 0 000-019 71 71 71 120 120 120 F F F
77532024-02-13T21:42:27.540Z 1 020-039 70 70 70 118 118 118 F F F
77542024-02-13T21:42:27.540Z 2 040-059 70 70 70 118 118 118 F F F
77552024-02-13T21:42:27.540Z 3 060-079 66 66 66 113 113 113 F F F
77562024-02-13T21:42:27.540Z 4 080-099 68 68 68 116 116 116 F F F
77572024-02-13T21:42:27.540Z 5 100-119 66 66 66 113 113 113 F F F
77582024-02-13T21:42:27.540Z 6 120-139 70 70 70 118 118 118 F F F
77592024-02-13T21:42:27.540Z 7 140-159 70 70 70 119 119 119 F F F
77602024-02-13T21:42:27.541Z 8 160-179 62 62 62 107 107 107 F F F
77612024-02-13T21:42:27.541Z 9 180-199 71 71 71 120 120 120 F F F
77622024-02-13T21:42:27.541Z 10 200-219 69 69 69 117 117 117 F F F
77632024-02-13T21:42:27.541Z 11 220-239 68 68 68 116 116 116 F F F
77642024-02-13T21:42:27.542Z 12 240-259 68 68 68 116 116 116 F F F
77652024-02-13T21:42:27.542Z 13 260-279 70 70 70 119 119 119 F F F
77662024-02-13T21:42:27.542Z 14 280-299 66 66 66 112 112 112 F F F
77672024-02-13T21:42:27.542Z 15 300-319 70 70 70 118 118 118 F F F
77682024-02-13T21:42:27.542Z 16 320-339 68 68 68 116 116 116 F F F
77692024-02-13T21:42:27.542Z 17 340-359 61 61 61 106 106 106 F F F
77702024-02-13T21:42:27.542Z 18 360-379 70 70 70 119 119 119 F F F
77712024-02-13T21:42:27.542Z 19 380-399 68 68 68 116 116 116 F F F
77722024-02-13T21:42:27.543Z 20 400-419 62 62 62 107 107 107 F F F
77732024-02-13T21:42:27.543Z 21 420-439 70 70 70 118 118 118 F F F
77742024-02-13T21:42:27.543Z 22 440-459 62 62 62 107 107 107 F F F
77752024-02-13T21:42:27.543Z 23 460-479 68 68 68 116 116 116 F F F
77762024-02-13T21:42:27.543Z 24 480-499 68 68 68 116 116 116 F F F
77772024-02-13T21:42:27.543Z 25 500-519 70 70 70 119 119 119 F F F
77782024-02-13T21:42:27.543Z 26 520-539 69 69 69 117 117 117 F F F
77792024-02-13T21:42:27.543Z 27 540-559 58 58 58 93 93 93 F F F
77802024-02-13T21:42:27.543Z 28 560-579 54 54 54 86 86 86 F F F
77812024-02-13T21:42:27.543Z 29 580-599 61 61 61 106 106 106 F F F
77822024-02-13T21:42:27.543ZMax gen: 71, Max flush: 120
77832024-02-13T21:42:27.557Z
77842024-02-13T21:42:27.557ZBegin loop 36
77852024-02-13T21:42:27.557ZDownstairs to restart: 2
77862024-02-13T21:42:27.557Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324551024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
77872024-02-13T21:42:27.558Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324827253Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
77882024-02-13T21:42:27.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324868809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
77892024-02-13T21:42:27.558Z{"msg":"Crucible 32cf5375-2f8c-4503-84e0-834c5899c0f2 has session id: 3e27ac61-032e-411e-872e-bd016d944eab","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324894929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426Crucible runtime is spawned
77902024-02-13T21:42:27.558Z,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77912024-02-13T21:42:27.558Z{"msg":"Upstairs opts: Upstairs UUID: 32cf5375-2f8c-4503-84e0-834c5899c0f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324953938Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77922024-02-13T21:42:27.558Z{"msg":"Crucible stats registered with UUID: 32cf5375-2f8c-4503-84e0-834c5899c0f2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.324994793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77932024-02-13T21:42:27.559Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325013246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
77942024-02-13T21:42:27.559Z{"msg":"32cf5375-2f8c-4503-84e0-834c5899c0f2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325077073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77952024-02-13T21:42:27.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325116229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77962024-02-13T21:42:27.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325132333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77972024-02-13T21:42:27.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325164641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77982024-02-13T21:42:27.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325182915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
77992024-02-13T21:42:27.560Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325200588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78002024-02-13T21:42:27.560Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325214383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78012024-02-13T21:42:27.560Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325276341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78022024-02-13T21:42:27.560Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325323593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78032024-02-13T21:42:27.560Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325349534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78042024-02-13T21:42:27.560Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325540895Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78052024-02-13T21:42:27.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.325592016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78062024-02-13T21:42:27.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.32566556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"io task","client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78072024-02-13T21:42:27.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326290533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78082024-02-13T21:42:27.561Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326329959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78092024-02-13T21:42:27.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326358398Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78102024-02-13T21:42:27.561Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326422235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78112024-02-13T21:42:27.562Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326930361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78122024-02-13T21:42:27.562Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.326976714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78132024-02-13T21:42:27.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327052436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78142024-02-13T21:42:27.562Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327099539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78152024-02-13T21:42:27.562Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327634845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78162024-02-13T21:42:27.562Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327666563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78172024-02-13T21:42:27.563Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327740047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78182024-02-13T21:42:27.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327775134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78192024-02-13T21:42:27.563Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327823137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78202024-02-13T21:42:27.563Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327859154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78212024-02-13T21:42:27.564Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.327874988Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78222024-02-13T21:42:27.564Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328096847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78232024-02-13T21:42:27.564Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328150998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78242024-02-13T21:42:27.564Z{"msg":"[0]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 119, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328184646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78252024-02-13T21:42:27.564Z{"msg":"[0]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 70, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.32820013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78262024-02-13T21:42:27.564Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328221742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78272024-02-13T21:42:27.564Z{"msg":"[1]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 119, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328235997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78282024-02-13T21:42:27.564Z{"msg":"[1]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 70, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.3282562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78292024-02-13T21:42:27.565Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328270864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78302024-02-13T21:42:27.565ZWait for a query_work_queue command to finish before sending IO
78312024-02-13T21:42:27.565Z{"msg":"[2]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 119, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328296755Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78322024-02-13T21:42:27.565Z{"msg":"[2]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 70, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.32831116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78332024-02-13T21:42:27.565Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328325544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78342024-02-13T21:42:27.565Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.32833865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78352024-02-13T21:42:27.565Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328352375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78362024-02-13T21:42:27.565Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328372927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78372024-02-13T21:42:27.566Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328387072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"":"downstairs","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78382024-02-13T21:42:27.566Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328401097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78392024-02-13T21:42:27.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328413453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"0","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78402024-02-13T21:42:27.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328434795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"1","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78412024-02-13T21:42:27.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.32844805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"client":"2","":"downstairs","session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78422024-02-13T21:42:27.566Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328467793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78432024-02-13T21:42:27.566Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328480658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78442024-02-13T21:42:27.567Z{"msg":"32cf5375-2f8c-4503-84e0-834c5899c0f2 is now active with session: 3e27ac61-032e-411e-872e-bd016d944eab","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328494233Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78452024-02-13T21:42:27.567Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328509038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426,"session_id":"3e27ac61-032e-411e-872e-bd016d944eab"}
78462024-02-13T21:42:27.567Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:26.328539277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
78472024-02-13T21:42:27.567ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78482024-02-13T21:42:27.567ZLoading write count information from file "/tmp/test_repair_verify.data"
78492024-02-13T21:42:27.567ZRun Repair workload
78502024-02-13T21:42:27.567Z01/30 Write block 316 len 20480 data: 7 6 5 7 7
78512024-02-13T21:42:27.568Z02/30 Read block 518 len 24576
78522024-02-13T21:42:27.568Z03/30 Read block 87 len 20480
78532024-02-13T21:42:27.573Z04/30 Write block 342 len 16384 data: 7 7 6 7
78542024-02-13T21:42:27.573Z05/30 Read block 312 len 40960
78552024-02-13T21:42:27.573Z06/30 Write block 343 len 16384 data: 8 7 8 7
78562024-02-13T21:42:27.573Z07/30 Read block 436 len 20480
78572024-02-13T21:42:27.573Z08/30 Flush
78582024-02-13T21:42:27.573Z09/30 Write block 325 len 28672 data: 8 8 8 5 4 4 8
78592024-02-13T21:42:27.573Z10/30 Read block 213 len 4096
78602024-02-13T21:42:27.574Z11/30 Read block 251 len 12288
78612024-02-13T21:42:27.574Z12/30 Flush
78622024-02-13T21:42:27.574Z13/30 Flush
78632024-02-13T21:42:27.574Z14/30 Write block 142 len 40960 data: 8 6 6 6 5 5 4 4 4 5
78642024-02-13T21:42:27.574Z15/30 Read block 536 len 12288
78652024-02-13T21:42:27.574Z16/30 Flush
78662024-02-13T21:42:27.574Z17/30 Write block 367 len 16384 data: 8 7 7 7
78672024-02-13T21:42:27.574Z18/30 Read block 516 len 36864
78682024-02-13T21:42:27.574Z19/30 Flush
78692024-02-13T21:42:27.574Z20/30 Write block 328 len 16384 data: 6 5 5 9
78702024-02-13T21:42:27.574Z21/30 Read block 290 len 28672
78712024-02-13T21:42:27.574Z22/30 Read block 143 len 36864
78722024-02-13T21:42:27.578Z23/30 Read block 173 len 4096
78732024-02-13T21:42:27.578Z24/30 Write block 502 len 40960 data: 5 4 6 6 8 7 7 7 8 8
78742024-02-13T21:42:27.578Z25/30 Flush
78752024-02-13T21:42:27.578Z26/30 Write block 432 len 16384 data: 10 7 8 8
78762024-02-13T21:42:27.578Z27/30 Write block 509 len 28672 data: 8 9 9 8 9 11 10
78772024-02-13T21:42:27.578Z28/30 Read block 530 len 36864
78782024-02-13T21:42:28.220Z29/30 Read block 524 len 40960
78792024-02-13T21:42:28.221Z30/30 Read block 458 len 40960
78802024-02-13T21:42:28.221Z----------------------------------------------------------------
78812024-02-13T21:42:28.221Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78822024-02-13T21:42:28.221ZStates: Active Active Active
78832024-02-13T21:42:28.221ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78842024-02-13T21:42:28.221Z 1 Acked 1000 Write 5 Done Done Done false
78852024-02-13T21:42:28.221Z 2 Acked 1001 Read 6 Done Done Done false
78862024-02-13T21:42:28.222Z 3 Acked 1002 Read 5 Done Done Done false
78872024-02-13T21:42:28.222Z 4 Acked 1003 Write 4 Done Done Done false
78882024-02-13T21:42:28.222Z 5 Acked 1004 Read 10 Done Done Done false
78892024-02-13T21:42:28.222Z 6 Acked 1005 Write 4 Done Done Done false
78902024-02-13T21:42:28.222Z 7 Acked 1006 Read 5 Done Done Sent false
78912024-02-13T21:42:28.222Z 8 Acked 1007 Flush 0 Done Done Sent false
78922024-02-13T21:42:28.222Z 9 Acked 1008 Write 7 Done Done Sent false
78932024-02-13T21:42:28.222Z 10 Acked 1009 Read 1 Done Done Sent false
78942024-02-13T21:42:28.222Z 11 Acked 1010 Read 3 Done Done Sent false
78952024-02-13T21:42:28.223Z 12 Acked 1011 Flush 0 Done Done Sent false
78962024-02-13T21:42:28.223Z 13 Acked 1012 Flush 0 Done Done Sent false
78972024-02-13T21:42:28.224Z 14 Acked 1013 Write 10 Done Done Sent false
78982024-02-13T21:42:28.224Z 15 Acked 1014 Read 3 Done Done Sent false
78992024-02-13T21:42:28.224Z 16 Acked 1015 Flush 0 Done Done Sent false
79002024-02-13T21:42:28.224Z 17 Acked 1016 Write 4 Done Done Sent false
79012024-02-13T21:42:28.224Z 18 Acked 1017 Read 9 Done Done Sent false
79022024-02-13T21:42:28.224Z 19 Acked 1018 Flush 0 Done Done Sent false
79032024-02-13T21:42:28.224Z 20 Acked 1019 Write 4 Done Done Sent false
79042024-02-13T21:42:28.224Z 21 Acked 1020 Read 7 Done Done Sent false
79052024-02-13T21:42:28.224Z 22 Acked 1021 Read 9 Done Done Sent false
79062024-02-13T21:42:28.225Z 23 Acked 1022 Read 1 Done Done Sent false
79072024-02-13T21:42:28.225Z 24 Acked 1023 Write 10 Done Done Sent false
79082024-02-13T21:42:28.225Z 25 Acked 1024 Flush 0 Done Done Sent false
79092024-02-13T21:42:28.225Z 26 Acked 1025 Write 4 Done Done Sent false
79102024-02-13T21:42:28.225Z 27 Acked 1026 Write 7 Done Done Sent false
79112024-02-13T21:42:28.225Z 28 Acked 1027 Read 9 Done Done Sent false
79122024-02-13T21:42:28.225Z 29 Acked 1028 Read 10 Done Done Sent false
79132024-02-13T21:42:28.225Z 30 Acked 1029 Read 10 Done Sent Sent false
79142024-02-13T21:42:28.225Z STATES DS:0 DS:1 DS:2 TOTAL
79152024-02-13T21:42:28.225Z New 0 0 0 0
79162024-02-13T21:42:28.226Z Sent 0 1 24 25
79172024-02-13T21:42:28.226Z Done 30 29 6 65
79182024-02-13T21:42:28.226Z Skipped 0 0 0 0
79192024-02-13T21:42:28.226Z Error 0 0 0 0
79202024-02-13T21:42:28.226ZLast Flush: 1024 1024 0
79212024-02-13T21:42:28.226ZDownstairs last five completed:
79222024-02-13T21:42:28.226ZUpstairs last five completed: 30 29 28 27 26
79232024-02-13T21:42:28.226Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:26.364911513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1426}
79242024-02-13T21:42:28.226ZWrote out file "/tmp/test_repair_verify.data"
79252024-02-13T21:42:28.232Z
79262024-02-13T21:42:28.233ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79272024-02-13T21:42:28.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79282024-02-13T21:42:28.233Z 0 000-019 71 71 71 120 120 120 F F F
79292024-02-13T21:42:28.233Z 1 020-039 70 70 70 118 118 118 F F F
79302024-02-13T21:42:28.233Z 2 040-059 70 70 70 118 118 118 F F F
79312024-02-13T21:42:28.233Z 3 060-079 66 66 66 113 113 113 F F F
79322024-02-13T21:42:28.234Z 4 080-099 68 68 68 116 116 116 F F F
79332024-02-13T21:42:28.234Z 5 100-119 66 66 66 113 113 113 F F F
79342024-02-13T21:42:28.234Z 6 120-139 70 70 70 118 118 118 F F F
79352024-02-13T21:42:28.234Z 7 140-159 72 72 70 124 124 119 F F F <---
79362024-02-13T21:42:28.234Z 8 160-179 62 62 62 107 107 107 F F F
79372024-02-13T21:42:28.235Z 9 180-199 71 71 71 120 120 120 F F F
79382024-02-13T21:42:28.235Z 10 200-219 69 69 69 117 117 117 F F F
79392024-02-13T21:42:28.235Z 11 220-239 68 68 68 116 116 116 F F F
79402024-02-13T21:42:28.235Z 12 240-259 68 68 68 116 116 116 F F F
79412024-02-13T21:42:28.235Z 13 260-279 70 70 70 119 119 119 F F F
79422024-02-13T21:42:28.235Z 14 280-299 66 66 66 112 112 112 F F F
79432024-02-13T21:42:28.235Z 15 300-319 72 72 70 121 121 118 F F T <---
79442024-02-13T21:42:28.236Z 16 320-339 72 72 68 126 126 116 F F T <---
79452024-02-13T21:42:28.236Z 17 340-359 72 72 61 121 121 106 F F T <---
79462024-02-13T21:42:28.236Z 18 360-379 72 72 70 125 125 119 F F F <---
79472024-02-13T21:42:28.236Z 19 380-399 68 68 68 116 116 116 F F F
79482024-02-13T21:42:28.236Z 20 400-419 62 62 62 107 107 107 F F F
79492024-02-13T21:42:28.236Z 21 420-439 70 70 70 118 118 118 T T F <---
79502024-02-13T21:42:28.236Z 22 440-459 62 62 62 107 107 107 F F F
79512024-02-13T21:42:28.236Z 23 460-479 68 68 68 116 116 116 F F F
79522024-02-13T21:42:28.237Z 24 480-499 68 68 68 116 116 116 F F F
79532024-02-13T21:42:28.237Z 25 500-519 72 72 70 126 126 119 T T F <---
79542024-02-13T21:42:28.237Z 26 520-539 69 69 69 117 117 117 F F F
79552024-02-13T21:42:28.237Z 27 540-559 58 58 58 93 93 93 F F F
79562024-02-13T21:42:28.238Z 28 560-579 54 54 54 86 86 86 F F F
79572024-02-13T21:42:28.238Z 29 580-599 61 61 61 106 106 106 F F F
79582024-02-13T21:42:28.238ZMax gen: 72, Max flush: 126
79592024-02-13T21:42:28.238ZError: Difference in extent metadata found!
79602024-02-13T21:42:28.238ZA Difference in extent metadata is expected here
79612024-02-13T21:42:28.238ZOn loop 36
79622024-02-13T21:42:28.238Z
79632024-02-13T21:42:28.239ZVerifying data now
79642024-02-13T21:42:33.008ZNo change after verify
79652024-02-13T21:42:33.023ZLoop: 36 Downstairs dump after verify (and repair):
79662024-02-13T21:42:33.440ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79672024-02-13T21:42:33.440Z 0 000-019 71 71 71 120 120 120 F F F
79682024-02-13T21:42:33.440Z 1 020-039 70 70 70 118 118 118 F F F
79692024-02-13T21:42:33.440Z 2 040-059 70 70 70 118 118 118 F F F
79702024-02-13T21:42:33.440Z 3 060-079 66 66 66 113 113 113 F F F
79712024-02-13T21:42:33.440Z 4 080-099 68 68 68 116 116 116 F F F
79722024-02-13T21:42:33.440Z 5 100-119 66 66 66 113 113 113 F F F
79732024-02-13T21:42:33.440Z 6 120-139 70 70 70 118 118 118 F F F
79742024-02-13T21:42:33.440Z 7 140-159 72 72 72 124 124 124 F F F
79752024-02-13T21:42:33.440Z 8 160-179 62 62 62 107 107 107 F F F
79762024-02-13T21:42:33.440Z 9 180-199 71 71 71 120 120 120 F F F
79772024-02-13T21:42:33.440Z 10 200-219 69 69 69 117 117 117 F F F
79782024-02-13T21:42:33.440Z 11 220-239 68 68 68 116 116 116 F F F
79792024-02-13T21:42:33.440Z 12 240-259 68 68 68 116 116 116 F F F
79802024-02-13T21:42:33.441Z 13 260-279 70 70 70 119 119 119 F F F
79812024-02-13T21:42:33.441Z 14 280-299 66 66 66 112 112 112 F F F
79822024-02-13T21:42:33.441Z 15 300-319 72 72 72 121 121 121 F F F
79832024-02-13T21:42:33.441Z 16 320-339 72 72 72 126 126 126 F F F
79842024-02-13T21:42:33.441Z 17 340-359 72 72 72 121 121 121 F F F
79852024-02-13T21:42:33.441Z 18 360-379 72 72 72 125 125 125 F F F
79862024-02-13T21:42:33.441Z 19 380-399 68 68 68 116 116 116 F F F
79872024-02-13T21:42:33.441Z 20 400-419 62 62 62 107 107 107 F F F
79882024-02-13T21:42:33.441Z 21 420-439 73 73 73 127 127 127 F F F
79892024-02-13T21:42:33.441Z 22 440-459 62 62 62 107 107 107 F F F
79902024-02-13T21:42:33.441Z 23 460-479 68 68 68 116 116 116 F F F
79912024-02-13T21:42:33.441Z 24 480-499 68 68 68 116 116 116 F F F
79922024-02-13T21:42:33.441Z 25 500-519 73 73 73 127 127 127 F F F
79932024-02-13T21:42:33.441Z 26 520-539 69 69 69 117 117 117 F F F
79942024-02-13T21:42:33.441Z 27 540-559 58 58 58 93 93 93 F F F
79952024-02-13T21:42:33.441Z 28 560-579 54 54 54 86 86 86 F F F
79962024-02-13T21:42:33.441Z 29 580-599 61 61 61 106 106 106 F F F
79972024-02-13T21:42:33.441ZMax gen: 73, Max flush: 127
79982024-02-13T21:42:33.441Z
79992024-02-13T21:42:33.441ZBegin loop 37
80002024-02-13T21:42:33.441ZDownstairs to restart: 1
80012024-02-13T21:42:33.469Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824501563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
80022024-02-13T21:42:33.470Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824796196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
80032024-02-13T21:42:33.470ZCrucible runtime is spawned
80042024-02-13T21:42:33.470Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824841819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
80052024-02-13T21:42:33.470Z{"msg":"Crucible 18fbdebe-f30f-4cb8-ae02-7d6ec7499321 has session id: 5c0e3011-ffdf-4b79-bd0f-961bebafa409","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824862112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80062024-02-13T21:42:33.470Z{"msg":"Upstairs opts: Upstairs UUID: 18fbdebe-f30f-4cb8-ae02-7d6ec7499321, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82489645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80072024-02-13T21:42:33.470Z{"msg":"Crucible stats registered with UUID: 18fbdebe-f30f-4cb8-ae02-7d6ec7499321","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824912604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80082024-02-13T21:42:33.470Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.824932167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
80092024-02-13T21:42:33.470Z{"msg":"18fbdebe-f30f-4cb8-ae02-7d6ec7499321 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825030091Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80102024-02-13T21:42:33.470Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825073455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80112024-02-13T21:42:33.470Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825099556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80122024-02-13T21:42:33.470Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825119379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80132024-02-13T21:42:33.470Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825139851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80142024-02-13T21:42:33.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825160694Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80152024-02-13T21:42:33.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825176788Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80162024-02-13T21:42:33.471Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825207107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80172024-02-13T21:42:33.471Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825237336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80182024-02-13T21:42:33.471Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825263177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80192024-02-13T21:42:33.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825284899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80202024-02-13T21:42:33.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.825429336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80212024-02-13T21:42:33.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.8254737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"io task","client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80222024-02-13T21:42:33.471Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.826046493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80232024-02-13T21:42:33.472Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.826086098Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80242024-02-13T21:42:33.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.826114348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80252024-02-13T21:42:33.472Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.826203346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80262024-02-13T21:42:33.472Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82663354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80272024-02-13T21:42:33.472Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.826791152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80282024-02-13T21:42:33.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827165816Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80292024-02-13T21:42:33.472Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827205922Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80302024-02-13T21:42:33.472Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827236801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80312024-02-13T21:42:33.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827263061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80322024-02-13T21:42:33.472Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827287902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80332024-02-13T21:42:33.472Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827320031Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80342024-02-13T21:42:33.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827342742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80352024-02-13T21:42:33.473Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827371262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80362024-02-13T21:42:33.473Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.827833644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80372024-02-13T21:42:33.473Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828276613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80382024-02-13T21:42:33.473Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82831313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80392024-02-13T21:42:33.473Z{"msg":"[0]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 124, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828330464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80402024-02-13T21:42:33.473Z{"msg":"[0]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 72, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828345639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80412024-02-13T21:42:33.473Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82836758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80422024-02-13T21:42:33.473Z{"msg":"[1]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 124, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828382035Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80432024-02-13T21:42:33.473Z{"msg":"[1]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 72, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828402598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80442024-02-13T21:42:33.473Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
80452024-02-13T21:42:33.473Z,"time":"2024-02-13T21:42:31.828417153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80462024-02-13T21:42:33.474Z{"msg":"[2]R flush_numbers[0..12]: [120, 118, 118, 113, 116, 113, 118, 124, 107, 120, 117, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828446282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80472024-02-13T21:42:33.474Z{"msg":"[2]R generation[0..12]: [71, 70, 70, 66, 68, 66, 70, 72, 62, 71, 69, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828466485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80482024-02-13T21:42:33.474Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82848081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80492024-02-13T21:42:33.474Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828500642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80502024-02-13T21:42:33.474Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828514437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80512024-02-13T21:42:33.474Z{"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828527433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80522024-02-13T21:42:33.474Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828546846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"":"downstairs","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80532024-02-13T21:42:33.474Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828560621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80542024-02-13T21:42:33.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828573496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"0","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80552024-02-13T21:42:33.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828586881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"1","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80562024-02-13T21:42:33.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828606354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"client":"2","":"downstairs","session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80572024-02-13T21:42:33.474Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828619799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80582024-02-13T21:42:33.475Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828632635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80592024-02-13T21:42:33.475Z{"msg":"18fbdebe-f30f-4cb8-ae02-7d6ec7499321 is now active with session: 5c0e3011-ffdf-4b79-bd0f-961bebafa409","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.82864552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80602024-02-13T21:42:33.475Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828663583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434,"session_id":"5c0e3011-ffdf-4b79-bd0f-961bebafa409"}
80612024-02-13T21:42:33.475Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:31.828676089Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
80622024-02-13T21:42:33.475ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80632024-02-13T21:42:33.475ZLoading write count information from file "/tmp/test_repair_verify.data"
80642024-02-13T21:42:33.475ZRun Repair workload
80652024-02-13T21:42:33.475Z01/30 Write block 41 len 12288 data: 7 5 4
80662024-02-13T21:42:33.475Z02/30 Write block 562 len 4096 data: 2
80672024-02-13T21:42:33.475Z03/30 Read block 53 len 20480
80682024-02-13T21:42:33.475Z04/30 Write block 218 len 4096 data: 10
80692024-02-13T21:42:33.475Z05/30 Read block 307 len 36864
80702024-02-13T21:42:33.475Z06/30 Read block 102 len 36864
80712024-02-13T21:42:33.475Z07/30 Read block 505 len 28672
80722024-02-13T21:42:33.475Z08/30 Read block 231 len 16384
80732024-02-13T21:42:33.475Z09/30 Read block 155 len 36864
80742024-02-13T21:42:33.476Z10/30 Flush
80752024-02-13T21:42:33.476Z11/30 Flush
80762024-02-13T21:42:33.476Z12/30 Write block 449 len 28672 data: 6 5 6 5 5 4 5
80772024-02-13T21:42:33.476Z13/30 Write block 594 len 16384 data: 5 5 5 4
80782024-02-13T21:42:33.476Z14/30 Read block 330 len 4096
80792024-02-13T21:42:33.476Z15/30 Write block 414 len 28672 data: 6 7 8 9 9 9 9
80802024-02-13T21:42:33.476Z16/30 Read block 128 len 28672
80812024-02-13T21:42:33.476Z17/30 Write block 322 len 12288 data: 6 8 7
80822024-02-13T21:42:33.476Z18/30 Write block 498 len 12288 data: 10 8 8
80832024-02-13T21:42:33.476Z19/30 Read block 46 len 8192
80842024-02-13T21:42:33.476Z20/30 Write block 82 len 4096 data: 6
80852024-02-13T21:42:33.476Z21/30 Write block 25 len 16384 data: 4 5 4 4
80862024-02-13T21:42:33.477Z22/30 Flush
80872024-02-13T21:42:33.477Z23/30 Write block 427 len 36864 data: 6 7 9 8 9 11 8 9 9
80882024-02-13T21:42:33.477Z24/30 Read block 562 len 8192
80892024-02-13T21:42:33.477Z25/30 Write block 523 len 40960 data: 10 7 5 4 4 5 4 4 3 3
80902024-02-13T21:42:33.477Z26/30 Write block 293 len 20480 data: 9 9 7 7 7
80912024-02-13T21:42:33.477Z27/30 Write block 14 len 12288 data: 10 11 13
80922024-02-13T21:42:33.477Z28/30 Write block 269 len 24576 data: 6 6 7 7 7 7
80932024-02-13T21:42:33.477Z29/30 Write block 25 len 4096 data: 5
80942024-02-13T21:42:33.477Z30/30 Read block 459 len 4096
80952024-02-13T21:42:33.477Z----------------------------------------------------------------
80962024-02-13T21:42:33.477Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80972024-02-13T21:42:33.477ZStates: Active Active Active
80982024-02-13T21:42:33.477ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80992024-02-13T21:42:33.477Z 1 Acked 1000 Write 3 Done Done Done false
81002024-02-13T21:42:33.478Z 2 Acked 1001 Write 1 Done Done Done false
81012024-02-13T21:42:33.498Z 3 Acked 1002 Read 5 Done Done Done false
81022024-02-13T21:42:33.499Z 4 Acked 1003 Write 1 Done Sent Done false
81032024-02-13T21:42:33.499Z 5 Acked 1004 Read 9 Done Sent Done false
81042024-02-13T21:42:33.499Z 6 Acked 1005 Read 9 Done Sent Done false
81052024-02-13T21:42:33.499Z 7 Acked 1006 Read 7 Done Sent Done false
81062024-02-13T21:42:33.499Z 8 Acked 1007 Read 4 Done Sent Done false
81072024-02-13T21:42:33.499Z 9 Acked 1008 Read 9 Done Sent Done false
81082024-02-13T21:42:33.500Z 10 Acked 1009 Flush 0 Done Sent Done false
81092024-02-13T21:42:33.500Z 11 Acked 1010 Flush 0 Done Sent Done false
81102024-02-13T21:42:33.500Z 12 Acked 1011 Write 7 Done Sent Done false
81112024-02-13T21:42:33.500Z 13 Acked 1012 Write 4 Done Sent Done false
81122024-02-13T21:42:33.500Z 14 Acked 1013 Read 1 Done Sent Done false
81132024-02-13T21:42:33.501Z 15 Acked 1014 Write 7 Done Sent Done false
81142024-02-13T21:42:33.501Z 16 Acked 1015 Read 7 Done Sent Done false
81152024-02-13T21:42:33.501Z 17 Acked 1016 Write 3 Done Sent Done false
81162024-02-13T21:42:33.501Z 18 Acked 1017 Write 3 Done Sent Done false
81172024-02-13T21:42:33.501Z 19 Acked 1018 Read 2 Done Sent Done false
81182024-02-13T21:42:33.501Z 20 Acked 1019 Write 1 Done Sent Done false
81192024-02-13T21:42:33.501Z 21 Acked 1020 Write 4 Done Sent Done false
81202024-02-13T21:42:33.502Z 22 Acked 1021 Flush 0 Done Sent Done false
81212024-02-13T21:42:33.502Z 23 Acked 1022 Write 9 Done Sent Done false
81222024-02-13T21:42:33.502Z 24 Acked 1023 Read 2 Done Sent Done false
81232024-02-13T21:42:33.502Z 25 Acked 1024 Write 10 Done Sent Done false
81242024-02-13T21:42:33.502Z 26 Acked 1025 Write 5 Done Sent Done false
81252024-02-13T21:42:33.503Z 27 Acked 1026 Write 3 Done Sent Done false
81262024-02-13T21:42:33.503Z 28 Acked 1027 Write 6 Done Sent Done false
81272024-02-13T21:42:33.503Z 29 Acked 1028 Write 1 Done Sent Done false
81282024-02-13T21:42:33.503Z 30 Acked 1029 Read 1 Done Sent Done false
81292024-02-13T21:42:33.503Z STATES DS:0 DS:1 DS:2 TOTAL
81302024-02-13T21:42:33.503Z New 0 0 0 0
81312024-02-13T21:42:33.503Z Sent 0 27 0 27
81322024-02-13T21:42:33.503Z Done 30 3 30 63
81332024-02-13T21:42:33.503Z Skipped 0 0 0 0
81342024-02-13T21:42:33.504Z Error 0 0 0 0
81352024-02-13T21:42:33.504ZLast Flush: 1021 0 1021
81362024-02-13T21:42:33.504ZDownstairs last five completed:
81372024-02-13T21:42:33.504ZUpstairs last five completed: 30 29 28 27 26
81382024-02-13T21:42:33.505Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:31.858770892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1434}
81392024-02-13T21:42:33.505ZWrote out file "/tmp/test_repair_verify.data"
81402024-02-13T21:42:33.505Z
81412024-02-13T21:42:33.506ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81422024-02-13T21:42:33.506ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81432024-02-13T21:42:33.506Z 0 000-019 71 71 71 120 120 120 T F T <---
81442024-02-13T21:42:33.506Z 1 020-039 74 70 74 130 118 130 T F T <---
81452024-02-13T21:42:33.506Z 2 040-059 74 70 74 128 118 128 F T F <---
81462024-02-13T21:42:33.506Z 3 060-079 66 66 66 113 113 113 F F F
81472024-02-13T21:42:33.506Z 4 080-099 74 68 74 130 116 130 F F F <---
81482024-02-13T21:42:33.507Z 5 100-119 66 66 66 113 113 113 F F F
81492024-02-13T21:42:33.507Z 6 120-139 70 70 70 118 118 118 F F F
81502024-02-13T21:42:33.507Z 7 140-159 72 72 72 124 124 124 F F F
81512024-02-13T21:42:33.507Z 8 160-179 62 62 62 107 107 107 F F F
81522024-02-13T21:42:33.507Z 9 180-199 71 71 71 120 120 120 F F F
81532024-02-13T21:42:33.508Z 10 200-219 74 69 74 128 117 128 F F F <---
81542024-02-13T21:42:33.508Z 11 220-239 68 68 68 116 116 116 F F F
81552024-02-13T21:42:33.508Z 12 240-259 68 68 68 116 116 116 F F F
81562024-02-13T21:42:33.509Z 13 260-279 70 70 70 119 119 119 T F T <---
81572024-02-13T21:42:33.509Z 14 280-299 66 66 66 112 112 112 T F T <---
81582024-02-13T21:42:33.509Z 15 300-319 72 72 72 121 121 121 F F F
81592024-02-13T21:42:33.509Z 16 320-339 74 72 74 130 126 130 F F F <---
81602024-02-13T21:42:33.509Z 17 340-359 72 72 72 121 121 121 F F F
81612024-02-13T21:42:33.509Z 18 360-379 72 72 72 125 125 125 F F F
81622024-02-13T21:42:33.509Z 19 380-399 68 68 68 116 116 116 F F F
81632024-02-13T21:42:33.509Z 20 400-419 74 62 74 130 107 130 F F F <---
81642024-02-13T21:42:33.509Z 21 420-439 74 73 74 130 127 130 T F T <---
81652024-02-13T21:42:33.509Z 22 440-459 74 62 74 130 107 130 F F F <---
81662024-02-13T21:42:33.509Z 23 460-479 68 68 68 116 116 116 F F F
81672024-02-13T21:42:33.510Z 24 480-499 74 68 74 130 116 130 F F F <---
81682024-02-13T21:42:33.510Z 25 500-519 74 73 74 130 127 130 F F F <---
81692024-02-13T21:42:33.510Z 26 520-539 69 69 69 117 117 117 T F T <---
81702024-02-13T21:42:33.510Z 27 540-559 58 58 58 93 93 93 F F F
81712024-02-13T21:42:33.510Z 28 560-579 74 54 74 128 86 128 F T F <---
81722024-02-13T21:42:33.510Z 29 580-599 74 61 74 130 106 130 F F F <---
81732024-02-13T21:42:33.510ZMax gen: 74, Max flush: 130
81742024-02-13T21:42:33.510ZError: Difference in extent metadata found!
81752024-02-13T21:42:33.510ZA Difference in extent metadata is expected here
81762024-02-13T21:42:33.511ZOn loop 37
81772024-02-13T21:42:33.511Z
81782024-02-13T21:42:33.511ZVerifying data now
81792024-02-13T21:42:38.869ZNo change after verify
81802024-02-13T21:42:38.870ZLoop: 37 Downstairs dump after verify (and repair):
81812024-02-13T21:42:38.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81822024-02-13T21:42:38.886Z 0 000-019 75 75 75 131 131 131 F F F
81832024-02-13T21:42:38.886Z 1 020-039 75 75 75 131 131 131 F F F
81842024-02-13T21:42:38.886Z 2 040-059 74 74 74 128 128 128 F F F
81852024-02-13T21:42:38.886Z 3 060-079 66 66 66 113 113 113 F F F
81862024-02-13T21:42:38.886Z 4 080-099 74 74 74 130 130 130 F F F
81872024-02-13T21:42:38.887Z 5 100-119 66 66 66 113 113 113 F F F
81882024-02-13T21:42:38.892Z 6 120-139 70 70 70 118 118 118 F F F
81892024-02-13T21:42:38.892Z 7 140-159 72 72 72 124 124 124 F F F
81902024-02-13T21:42:38.892Z 8 160-179 62 62 62 107 107 107 F F F
81912024-02-13T21:42:38.892Z 9 180-199 71 71 71 120 120 120 F F F
81922024-02-13T21:42:38.892Z 10 200-219 74 74 74 128 128 128 F F F
81932024-02-13T21:42:38.893Z 11 220-239 68 68 68 116 116 116 F F F
81942024-02-13T21:42:38.893Z 12 240-259 68 68 68 116 116 116 F F F
81952024-02-13T21:42:38.893Z 13 260-279 75 75 75 131 131 131 F F F
81962024-02-13T21:42:38.893Z 14 280-299 75 75 75 131 131 131 F F F
81972024-02-13T21:42:38.893Z 15 300-319 72 72 72 121 121 121 F F F
81982024-02-13T21:42:38.893Z 16 320-339 74 74 74 130 130 130 F F F
81992024-02-13T21:42:38.893Z 17 340-359 72 72 72 121 121 121 F F F
82002024-02-13T21:42:38.893Z 18 360-379 72 72 72 125 125 125 F F F
82012024-02-13T21:42:38.893Z 19 380-399 68 68 68 116 116 116 F F F
82022024-02-13T21:42:38.893Z 20 400-419 74 74 74 130 130 130 F F F
82032024-02-13T21:42:38.893Z 21 420-439 75 75 75 131 131 131 F F F
82042024-02-13T21:42:38.893Z 22 440-459 74 74 74 130 130 130 F F F
82052024-02-13T21:42:38.893Z 23 460-479 68 68 68 116 116 116 F F F
82062024-02-13T21:42:38.893Z 24 480-499 74 74 74 130 130 130 F F F
82072024-02-13T21:42:38.893Z 25 500-519 74 74 74 130 130 130 F F F
82082024-02-13T21:42:38.894Z 26 520-539 75 75 75 131 131 131 F F F
82092024-02-13T21:42:38.894Z 27 540-559 58 58 58 93 93 93 F F F
82102024-02-13T21:42:38.894Z 28 560-579 74 74 74 128 128 128 F F F
82112024-02-13T21:42:38.894Z 29 580-599 74 74 74 130 130 130 F F F
82122024-02-13T21:42:38.894ZMax gen: 75, Max flush: 131
82132024-02-13T21:42:38.894Z
82142024-02-13T21:42:38.894ZBegin loop 38
82152024-02-13T21:42:38.894ZDownstairs to restart: 0
82162024-02-13T21:42:38.907Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684500149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
82172024-02-13T21:42:38.907Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684756157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
82182024-02-13T21:42:38.907Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684784317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
82192024-02-13T21:42:38.907ZCrucible runtime is spawned
82202024-02-13T21:42:38.914Z{"msg":"Crucible bed00dbb-167f-411c-a499-dd2133db6c7f has session id: 9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684815805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82212024-02-13T21:42:38.914Z{"msg":"Upstairs opts: Upstairs UUID: bed00dbb-167f-411c-a499-dd2133db6c7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684859949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82222024-02-13T21:42:38.914Z{"msg":"Crucible stats registered with UUID: bed00dbb-167f-411c-a499-dd2133db6c7f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684888389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82232024-02-13T21:42:38.914Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.684920947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
82242024-02-13T21:42:38.915Z{"msg":"bed00dbb-167f-411c-a499-dd2133db6c7f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685032816Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82252024-02-13T21:42:38.915Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685058817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82262024-02-13T21:42:38.915Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685073761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82272024-02-13T21:42:38.915Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685103621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82282024-02-13T21:42:38.915Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685125093Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82292024-02-13T21:42:38.915Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.68516035Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82302024-02-13T21:42:38.915Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685186471Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82312024-02-13T21:42:38.916Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685209063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82322024-02-13T21:42:38.916Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685254366Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82332024-02-13T21:42:38.917Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685293572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82342024-02-13T21:42:38.917Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685323981Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82352024-02-13T21:42:38.917Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685359958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82362024-02-13T21:42:38.918Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.685445627Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"io task","client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82372024-02-13T21:42:38.918Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686158508Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82382024-02-13T21:42:38.918Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686205531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82392024-02-13T21:42:38.918Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686231732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82402024-02-13T21:42:38.918Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686265399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82412024-02-13T21:42:38.918Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686830214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82422024-02-13T21:42:38.918Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.686876507Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82432024-02-13T21:42:38.918Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687113521Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82442024-02-13T21:42:38.918Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687152298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82452024-02-13T21:42:38.918Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687388352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82462024-02-13T21:42:38.918Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687432936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82472024-02-13T21:42:38.919Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687459686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82482024-02-13T21:42:38.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687575064Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82492024-02-13T21:42:38.919Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687602534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82502024-02-13T21:42:38.919Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687617009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82512024-02-13T21:42:38.919Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.687784048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82522024-02-13T21:42:38.919Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688219431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82532024-02-13T21:42:38.919Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688257157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82542024-02-13T21:42:38.919Z{"msg":"[0]R flush_numbers[0..12]: [131, 131, 128, 113, 130, 113, 118, 124, 107, 120, 128, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688274021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82552024-02-13T21:42:38.919Z{"msg":"[0]R generation[0..12]: [75, 75, 74, 66, 74, 66, 70, 72, 62, 71, 74, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688296212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82562024-02-13T21:42:38.919Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688311037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82572024-02-13T21:42:38.919Z{"msg":"[1]R flush_numbers[0..12]: [131, 131, 128, 113, 130, 113, 118, 124, 107, 120, 128, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688325332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82582024-02-13T21:42:38.919Z{"msg":"[1]R generation[0..12]: [75, 75, 74, 66, 74, 66, 70, 72, 62, 71, 74, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688352392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82592024-02-13T21:42:38.919ZWait for a query_work_queue command to finish before sending IO
82602024-02-13T21:42:38.919Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688391088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82612024-02-13T21:42:38.919Z{"msg":"[2]R flush_numbers[0..12]: [131, 131, 128, 113, 130, 113, 118, 124, 107, 120, 128, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688427585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82622024-02-13T21:42:38.919Z{"msg":"[2]R generation[0..12]: [75, 75, 74, 66, 74, 66, 70, 72, 62, 71, 74, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688458764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82632024-02-13T21:42:38.919Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.68849556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82642024-02-13T21:42:38.919Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688539334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82652024-02-13T21:42:38.919Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688566205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82662024-02-13T21:42:38.919Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688597633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82672024-02-13T21:42:38.919Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688621985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"":"downstairs","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82682024-02-13T21:42:38.920Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688649075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82692024-02-13T21:42:38.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688671267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"0","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82702024-02-13T21:42:38.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688699306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"1","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82712024-02-13T21:42:38.920Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688722838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"client":"2","":"downstairs","session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82722024-02-13T21:42:38.920Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688747699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82732024-02-13T21:42:38.920Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688769611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82742024-02-13T21:42:38.920Z{"msg":"bed00dbb-167f-411c-a499-dd2133db6c7f is now active with session: 9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688792483Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82752024-02-13T21:42:38.920Z9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82762024-02-13T21:42:38.920Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.68882764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442,"session_id":"9cf88cee-8f12-49d2-a05a-8c7d8a8a1b2f"}
82772024-02-13T21:42:38.920Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:37.688849902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
82782024-02-13T21:42:38.920ZLoading write count information from file "/tmp/test_repair_verify.data"
82792024-02-13T21:42:38.920ZRun Repair workload
82802024-02-13T21:42:38.920Z01/30 Write block 185 len 36864 data: 9 8 7 9 10 10 9 8 8
82812024-02-13T21:42:38.924Z02/30 Write block 516 len 36864 data: 9 11 13 16 17 13 12 11 8
82822024-02-13T21:42:38.924Z03/30 Flush
82832024-02-13T21:42:38.925Z04/30 Write block 552 len 40960 data: 6 6 6 5 6 5 5 3 2 2
82842024-02-13T21:42:38.925Z05/30 Write block 69 len 16384 data: 8 8 7 9
82852024-02-13T21:42:38.925Z06/30 Read block 401 len 36864
82862024-02-13T21:42:38.926Z07/30 Flush
82872024-02-13T21:42:38.926Z08/30 Write block 16 len 40960 data: 14 12 12 9 8 8 9 6 6 6
82882024-02-13T21:42:38.926Z09/30 Read block 137 len 36864
82892024-02-13T21:42:38.931Z10/30 Write block 411 len 4096 data: 7
82902024-02-13T21:42:38.931Z11/30 Read block 218 len 16384
82912024-02-13T21:42:38.931Z12/30 Read block 38 len 24576
82922024-02-13T21:42:38.931Z13/30 Read block 141 len 8192
82932024-02-13T21:42:38.931Z14/30 Read block 591 len 4096
82942024-02-13T21:42:38.931Z15/30 Read block 93 len 12288
82952024-02-13T21:42:38.932Z16/30 Read block 347 len 36864
82962024-02-13T21:42:38.949Z17/30 Write block 13 len 8192 data: 9 11
82972024-02-13T21:42:38.949Z18/30 Read block 259 len 20480
82982024-02-13T21:42:38.949Z19/30 Write block 532 len 12288 data: 4 5 6
82992024-02-13T21:42:38.949Z20/30 Write block 298 len 8192 data: 7 6
83002024-02-13T21:42:38.950Z21/30 Write block 200 len 16384 data: 9 8 8 7
83012024-02-13T21:42:38.950Z22/30 Write block 325 len 16384 data: 9 9 9 7
83022024-02-13T21:42:38.950Z23/30 Write block 546 len 24576 data: 8 7 5 7 7 8
83032024-02-13T21:42:38.950Z24/30 Write block 347 len 32768 data: 8 7 5 5 7 8 8 7
83042024-02-13T21:42:38.951Z25/30 Write block 332 len 8192 data: 10 10
83052024-02-13T21:42:38.951Z26/30 Read block 90 len 24576
83062024-02-13T21:42:38.951Z27/30 Write block 567 len 12288 data: 7 8 6
83072024-02-13T21:42:38.951Z28/30 Read block 306 len 24576
83082024-02-13T21:42:38.951Z29/30 Read block 238 len 24576
83092024-02-13T21:42:38.951Z30/30 Write block 473 len 40960 data: 10 10 9 6 5 6 6 6 6 7
83102024-02-13T21:42:38.951Z----------------------------------------------------------------
83112024-02-13T21:42:38.952Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:27
83122024-02-13T21:42:38.952ZStates: Active Active Active
83132024-02-13T21:42:38.952ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
83142024-02-13T21:42:38.952Z 4 Acked 1003 Write 10 Sent Done Done false
83152024-02-13T21:42:38.952Z 5 Acked 1004 Write 4 Sent Done Done false
83162024-02-13T21:42:38.952Z 6 Acked 1005 Read 9 Sent Done Done false
83172024-02-13T21:42:38.952Z 7 Acked 1006 Flush 0 Sent Done Done false
83182024-02-13T21:42:38.953Z 8 Acked 1007 Write 10 Sent Done Done false
83192024-02-13T21:42:38.953Z 9 Acked 1008 Read 9 Sent Done Done false
83202024-02-13T21:42:38.953Z 10 Acked 1009 Write 1 Sent Done Done false
83212024-02-13T21:42:38.953Z 11 Acked 1010 Read 4 Sent Done Done false
83222024-02-13T21:42:38.953Z 12 Acked 1011 Read 6 Sent Done Done false
83232024-02-13T21:42:38.954Z 13 Acked 1012 Read 2 Sent Done Done false
83242024-02-13T21:42:38.954Z 14 Acked 1013 Read 1 Sent Done Done false
83252024-02-13T21:42:38.954Z 15 Acked 1014 Read 3 Sent Done Done false
83262024-02-13T21:42:38.954Z 16 Acked 1015 Read 9 Sent Done Done false
83272024-02-13T21:42:38.954Z 17 Acked 1016 Write 2 Sent Done Done false
83282024-02-13T21:42:38.954Z 18 Acked 1017 Read 5 Sent Done Done false
83292024-02-13T21:42:38.954Z 19 Acked 1018 Write 3 Sent Done Done false
83302024-02-13T21:42:38.954Z 20 Acked 1019 Write 2 Sent Done Done false
83312024-02-13T21:42:38.955Z 21 Acked 1020 Write 4 Sent Done Done false
83322024-02-13T21:42:38.955Z 22 Acked 1021 Write 4 Sent Done Done false
83332024-02-13T21:42:38.955Z 23 Acked 1022 Write 6 Sent Done Done false
83342024-02-13T21:42:38.955Z 24 Acked 1023 Write 8 Sent Done Done false
83352024-02-13T21:42:38.955Z 25 Acked 1024 Write 2 Sent Done Done false
83362024-02-13T21:42:38.956Z 26 Acked 1025 Read 6 Sent Done Done false
83372024-02-13T21:42:38.956Z 27 Acked 1026 Write 3 Sent Done Done false
83382024-02-13T21:42:38.956Z 28 Acked 1027 Read 6 Sent Done Done false
83392024-02-13T21:42:38.956Z 29 Acked 1028 Read 6 Sent Done Done false
83402024-02-13T21:42:38.956Z 30 Acked 1029 Write 10 Sent Sent Sent false
83412024-02-13T21:42:38.956Z STATES DS:0 DS:1 DS:2 TOTAL
83422024-02-13T21:42:38.957Z New 0 0 0 0
83432024-02-13T21:42:38.957Z Sent 27 1 1 29
83442024-02-13T21:42:38.957Z Done 0 26 26 52
83452024-02-13T21:42:38.957Z Skipped 0 0 0 0
83462024-02-13T21:42:38.957Z Error 0 0 0 0
83472024-02-13T21:42:38.957ZLast Flush: 1002 1006 1006
83482024-02-13T21:42:38.958ZDownstairs last five completed: 1002 1001 1000
83492024-02-13T21:42:38.958ZUpstairs last five completed: 30 29 28 27 26
83502024-02-13T21:42:38.958Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:37.718846851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1442}
83512024-02-13T21:42:38.958ZWrote out file "/tmp/test_repair_verify.data"
83522024-02-13T21:42:38.973Z
83532024-02-13T21:42:38.974ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83542024-02-13T21:42:38.992ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83552024-02-13T21:42:38.993Z 0 000-019 75 75 75 131 131 131 F T T <---
83562024-02-13T21:42:38.993Z 1 020-039 75 75 75 131 131 131 F T T <---
83572024-02-13T21:42:38.993Z 2 040-059 74 74 74 128 128 128 F F F
83582024-02-13T21:42:38.993Z 3 060-079 66 76 76 113 133 133 F F F <---
83592024-02-13T21:42:38.993Z 4 080-099 74 74 74 130 130 130 F F F
83602024-02-13T21:42:38.993Z 5 100-119 66 66 66 113 113 113 F F F
83612024-02-13T21:42:38.993Z 6 120-139 70 70 70 118 118 118 F F F
83622024-02-13T21:42:38.993Z 7 140-159 72 72 72 124 124 124 F F F
83632024-02-13T21:42:38.994Z 8 160-179 62 62 62 107 107 107 F F F
83642024-02-13T21:42:38.994Z 9 180-199 76 76 76 132 132 132 F F F
83652024-02-13T21:42:38.994Z 10 200-219 74 74 74 128 128 128 F T T <---
83662024-02-13T21:42:38.994Z 11 220-239 68 68 68 116 116 116 F F F
83672024-02-13T21:42:38.994Z 12 240-259 68 68 68 116 116 116 F F F
83682024-02-13T21:42:38.994Z 13 260-279 75 75 75 131 131 131 F F F
83692024-02-13T21:42:38.994Z 14 280-299 75 75 75 131 131 131 F T T <---
83702024-02-13T21:42:38.994Z 15 300-319 72 72 72 121 121 121 F F F
83712024-02-13T21:42:38.995Z 16 320-339 74 74 74 130 130 130 F T T <---
83722024-02-13T21:42:38.995Z 17 340-359 72 72 72 121 121 121 F T T <---
83732024-02-13T21:42:38.995Z 18 360-379 72 72 72 125 125 125 F F F
83742024-02-13T21:42:38.995Z 19 380-399 68 68 68 116 116 116 F F F
83752024-02-13T21:42:38.996Z 20 400-419 74 74 74 130 130 130 F T T <---
83762024-02-13T21:42:38.996Z 21 420-439 75 75 75 131 131 131 F F F
83772024-02-13T21:42:38.996Z 22 440-459 74 74 74 130 130 130 F F F
83782024-02-13T21:42:38.996Z 23 460-479 68 68 68 116 116 116 F T T <---
83792024-02-13T21:42:38.996Z 24 480-499 74 74 74 130 130 130 F T T <---
83802024-02-13T21:42:38.996Z 25 500-519 76 76 76 132 132 132 F F F
83812024-02-13T21:42:38.996Z 26 520-539 76 76 76 132 132 132 F T T <---
83822024-02-13T21:42:38.997Z 27 540-559 58 76 76 93 133 133 F T T <---
83832024-02-13T21:42:38.997Z 28 560-579 74 76 76 128 133 133 F T T <---
83842024-02-13T21:42:38.997Z 29 580-599 74 74 74 130 130 130 F F F
83852024-02-13T21:42:38.997ZMax gen: 76, Max flush: 133
83862024-02-13T21:42:38.997ZError: Difference in extent metadata found!
83872024-02-13T21:42:38.998ZA Difference in extent metadata is expected here
83882024-02-13T21:42:38.998ZOn loop 38
83892024-02-13T21:42:38.998Z
83902024-02-13T21:42:38.999ZVerifying data now
83912024-02-13T21:42:43.629ZNo change after verify
83922024-02-13T21:42:43.629ZLoop: 38 Downstairs dump after verify (and repair):
83932024-02-13T21:42:43.646ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83942024-02-13T21:42:43.646Z 0 000-019 77 77 77 134 134 134 F F F
83952024-02-13T21:42:43.646Z 1 020-039 77 77 77 134 134 134 F F F
83962024-02-13T21:42:43.646Z 2 040-059 74 74 74 128 128 128 F F F
83972024-02-13T21:42:43.646Z 3 060-079 76 76 76 133 133 133 F F F
83982024-02-13T21:42:43.646Z 4 080-099 74 74 74 130 130 130 F F F
83992024-02-13T21:42:43.647Z 5 100-119 66 66 66 113 113 113 F F F
84002024-02-13T21:42:43.647Z 6 120-139 70 70 70 118 118 118 F F F
84012024-02-13T21:42:43.647Z 7 140-159 72 72 72 124 124 124 F F F
84022024-02-13T21:42:43.647Z 8 160-179 62 62 62 107 107 107 F F F
84032024-02-13T21:42:43.652Z 9 180-199 76 76 76 132 132 132 F F F
84042024-02-13T21:42:43.652Z 10 200-219 77 77 77 134 134 134 F F F
84052024-02-13T21:42:43.652Z 11 220-239 68 68 68 116 116 116 F F F
84062024-02-13T21:42:43.652Z 12 240-259 68 68 68 116 116 116 F F F
84072024-02-13T21:42:43.652Z 13 260-279 75 75 75 131 131 131 F F F
84082024-02-13T21:42:43.652Z 14 280-299 77 77 77 134 134 134 F F F
84092024-02-13T21:42:43.652Z 15 300-319 72 72 72 121 121 121 F F F
84102024-02-13T21:42:43.652Z 16 320-339 77 77 77 134 134 134 F F F
84112024-02-13T21:42:43.652Z 17 340-359 77 77 77 134 134 134 F F F
84122024-02-13T21:42:43.653Z 18 360-379 72 72 72 125 125 125 F F F
84132024-02-13T21:42:43.653Z 19 380-399 68 68 68 116 116 116 F F F
84142024-02-13T21:42:43.653Z 20 400-419 77 77 77 134 134 134 F F F
84152024-02-13T21:42:43.653Z 21 420-439 75 75 75 131 131 131 F F F
84162024-02-13T21:42:43.653Z 22 440-459 74 74 74 130 130 130 F F F
84172024-02-13T21:42:43.653Z 23 460-479 77 77 77 134 134 134 F F F
84182024-02-13T21:42:43.653Z 24 480-499 77 77 77 134 134 134 F F F
84192024-02-13T21:42:43.653Z 25 500-519 76 76 76 132 132 132 F F F
84202024-02-13T21:42:43.653Z 26 520-539 77 77 77 134 134 134 F F F
84212024-02-13T21:42:43.653Z 27 540-559 77 77 77 134 134 134 F F F
84222024-02-13T21:42:43.654Z 28 560-579 77 77 77 134 134 134 F F F
84232024-02-13T21:42:43.654Z 29 580-599 74 74 74 130 130 130 F F F
84242024-02-13T21:42:43.654ZMax gen: 77, Max flush: 134
84252024-02-13T21:42:43.654Z
84262024-02-13T21:42:43.654ZBegin loop 39
84272024-02-13T21:42:43.654ZDownstairs to restart: 0
84282024-02-13T21:42:43.666Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441538691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
84292024-02-13T21:42:43.666Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441824017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
84302024-02-13T21:42:43.666Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441850918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
84312024-02-13T21:42:44.096Z{"msg":"Crucible e463754c-f36f-4fab-9442-212ba1992ee9 has session id: 7c804610-2798-489f-a052-e883d656a2df","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441880157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":Crucible runtime is spawned
84322024-02-13T21:42:44.096Z1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84332024-02-13T21:42:44.096Z{"msg":"Upstairs opts: Upstairs UUID: e463754c-f36f-4fab-9442-212ba1992ee9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.44192604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84342024-02-13T21:42:44.096Z{"msg":"Crucible stats registered with UUID: e463754c-f36f-4fab-9442-212ba1992ee9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441952061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84352024-02-13T21:42:44.096Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.441989847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
84362024-02-13T21:42:44.096Z{"msg":"e463754c-f36f-4fab-9442-212ba1992ee9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442115042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84372024-02-13T21:42:44.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442142002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84382024-02-13T21:42:44.096Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442157666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84392024-02-13T21:42:44.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442178859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84402024-02-13T21:42:44.096Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442193373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84412024-02-13T21:42:44.096Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442211836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84422024-02-13T21:42:44.096Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442245104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84432024-02-13T21:42:44.096Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442329224Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84442024-02-13T21:42:44.097Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.44239613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84452024-02-13T21:42:44.097Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442427658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84462024-02-13T21:42:44.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442458727Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84472024-02-13T21:42:44.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442488836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84482024-02-13T21:42:44.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.442529431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"io task","client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84492024-02-13T21:42:44.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443098855Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84502024-02-13T21:42:44.097Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443141289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84512024-02-13T21:42:44.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443170969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84522024-02-13T21:42:44.097Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443199378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84532024-02-13T21:42:44.097Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443713052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84542024-02-13T21:42:44.097Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.443792753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84552024-02-13T21:42:44.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444169386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84562024-02-13T21:42:44.097Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444206993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84572024-02-13T21:42:44.098Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444335976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84582024-02-13T21:42:44.098Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444370134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84592024-02-13T21:42:44.099Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444385828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84602024-02-13T21:42:44.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444398983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84612024-02-13T21:42:44.099Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444413158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84622024-02-13T21:42:44.099Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444433491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84632024-02-13T21:42:44.099Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.444768659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84642024-02-13T21:42:44.099Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445173822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84652024-02-13T21:42:44.100Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445212568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84662024-02-13T21:42:44.100Z{"msg":"[0]R flush_numbers[0..12]: [134, 134, 128, 133, 130, 113, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445229762Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84672024-02-13T21:42:44.101Z{"msg":"[0]R generation[0..12]: [77, 77, 74, 76, 74, 66, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445252104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84682024-02-13T21:42:44.101Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445267088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84692024-02-13T21:42:44.101Z{"msg":"[1]R flush_numbers[0..12]: [134, 134, 128, 133, 130, 113, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445287771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84702024-02-13T21:42:44.101Z{"msg":"[1]R generation[0..12]: [77, 77, 74, 76, 74, 66, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445301866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84712024-02-13T21:42:44.102Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445322558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84722024-02-13T21:42:44.102Z{"msg":"[2]R flush_numbers[0..12]: [134, 134, 128, 133, 130, 113, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445336873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84732024-02-13T21:42:44.102ZWait for a query_work_queue command to finish before sending IO
84742024-02-13T21:42:44.102Z{"msg":"[2]R generation[0..12]: [77, 77, 74, 76, 74, 66, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445361704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84752024-02-13T21:42:44.102Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445376639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84762024-02-13T21:42:44.102Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445390034Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84772024-02-13T21:42:44.102Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445403379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84782024-02-13T21:42:44.102Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445423542Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84792024-02-13T21:42:44.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445436947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"":"downstairs","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84802024-02-13T21:42:44.103Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.44545683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84812024-02-13T21:42:44.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445469235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"0","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84822024-02-13T21:42:44.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445490247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"1","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84832024-02-13T21:42:44.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445503602Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"client":"2","":"downstairs","session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84842024-02-13T21:42:44.103Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445523585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84852024-02-13T21:42:44.104Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445536311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84862024-02-13T21:42:44.104Z{"msg":"e463754c-f36f-4fab-9442-212ba1992ee9 is now active with session: 7c804610-2798-489f-a052-e883d656a2df","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445549786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84872024-02-13T21:42:44.104Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445562071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450,"session_id":"7c804610-2798-489f-a052-e883d656a2df"}
84882024-02-13T21:42:44.104Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:42.445582304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
84892024-02-13T21:42:44.104ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84902024-02-13T21:42:44.104ZLoading write count information from file "/tmp/test_repair_verify.data"
84912024-02-13T21:42:44.104ZRun Repair workload
84922024-02-13T21:42:44.104Z01/30 Write block 507 len 36864 data: 8 8 9 10 10 9 10 12 11
84932024-02-13T21:42:44.524Z02/30 Write block 560 len 40960 data: 3 3 3 3 6 7 7 8 9 7
84942024-02-13T21:42:44.524Z03/30 Read block 559 len 16384
84952024-02-13T21:42:44.524Z04/30 Write block 380 len 12288 data: 7 9 9
84962024-02-13T21:42:44.524Z05/30 Read block 365 len 16384
84972024-02-13T21:42:44.524Z06/30 Read block 281 len 40960
84982024-02-13T21:42:44.524Z07/30 Read block 496 len 4096
84992024-02-13T21:42:44.524Z08/30 Write block 249 len 4096 data: 4
85002024-02-13T21:42:44.524Z09/30 Write block 478 len 32768 data: 7 7 7 7 8 6 5 5
85012024-02-13T21:42:44.524Z10/30 Read block 145 len 12288
85022024-02-13T21:42:44.524Z11/30 Read block 173 len 32768
85032024-02-13T21:42:44.524Z12/30 Flush
85042024-02-13T21:42:44.524Z13/30 Write block 96 len 28672 data: 6 7 8 7 7 9 13
85052024-02-13T21:42:44.524Z14/30 Write block 338 len 28672 data: 8 7 8 7 8 9 8
85062024-02-13T21:42:44.524Z15/30 Write block 251 len 32768 data: 6 5 5 7 7 7 7 6
85072024-02-13T21:42:44.524Z16/30 Read block 493 len 32768
85082024-02-13T21:42:44.524Z17/30 Write block 279 len 36864 data: 9 8 4 3 3 3 4 4 4
85092024-02-13T21:42:44.524Z18/30 Write block 274 len 20480 data: 8 9 9 11 9
85102024-02-13T21:42:44.524Z19/30 Write block 385 len 24576 data: 5 6 7 7 6 6
85112024-02-13T21:42:44.525Z20/30 Read block 161 len 12288
85122024-02-13T21:42:44.525Z21/30 Write block 55 len 24576 data: 5 6 7 7 8 7
85132024-02-13T21:42:44.525Z22/30 Read block 73 len 40960
85142024-02-13T21:42:44.525Z23/30 Read block 267 len 12288
85152024-02-13T21:42:44.525Z24/30 Write block 413 len 28672 data: 5 7 8 9 10 10 10
85162024-02-13T21:42:44.525Z25/30 Write block 445 len 24576 data: 7 7 6 6 7 6
85172024-02-13T21:42:44.525Z26/30 Write block 378 len 8192 data: 7 7
85182024-02-13T21:42:44.525Z27/30 Read block 145 len 24576
85192024-02-13T21:42:44.525Z28/30 Write block 423 len 28672 data: 7 6 6 8 7 8 10
85202024-02-13T21:42:44.525Z29/30 Write block 555 len 8192 data: 6 7
85212024-02-13T21:42:44.525Z30/30 Write block 107 len 8192 data: 6 6
85222024-02-13T21:42:44.525Z----------------------------------------------------------------
85232024-02-13T21:42:44.525Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
85242024-02-13T21:42:44.525ZStates: Active Active Active
85252024-02-13T21:42:44.525ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85262024-02-13T21:42:44.525Z 1 Acked 1000 Write 9 Sent Done Done false
85272024-02-13T21:42:44.525Z 2 Acked 1001 Write 10 Sent Done Done false
85282024-02-13T21:42:44.525Z 3 Acked 1002 Read 4 Sent Done Done false
85292024-02-13T21:42:44.525Z 4 Acked 1003 Write 3 Sent Done Done false
85302024-02-13T21:42:44.525Z 5 Acked 1004 Read 4 Sent Done Done false
85312024-02-13T21:42:44.525Z 6 Acked 1005 Read 10 Sent Done Done false
85322024-02-13T21:42:44.525Z 7 Acked 1006 Read 1 Sent Done Done false
85332024-02-13T21:42:44.525Z 8 Acked 1007 Write 1 Sent Done Done false
85342024-02-13T21:42:44.525Z 9 Acked 1008 Write 8 Sent Done Done false
85352024-02-13T21:42:44.525Z 10 Acked 1009 Read 3 Sent Done Done false
85362024-02-13T21:42:44.526Z 11 Acked 1010 Read 8 Sent Done Done false
85372024-02-13T21:42:44.526Z 12 Acked 1011 Flush 0 Sent Done Done false
85382024-02-13T21:42:44.526Z 13 Acked 1012 Write 7 Sent Done Done false
85392024-02-13T21:42:44.526Z 14 Acked 1013 Write 7 Sent Done Done false
85402024-02-13T21:42:44.526Z 15 Acked 1014 Write 8 Sent Done Done false
85412024-02-13T21:42:44.526Z 16 Acked 1015 Read 8 Sent Done Done false
85422024-02-13T21:42:44.526Z 17 Acked 1016 Write 9 Sent Done Done false
85432024-02-13T21:42:44.526Z 18 Acked 1017 Write 5 Sent Done Done false
85442024-02-13T21:42:44.526Z 19 Acked 1018 Write 6 Sent Done Done false
85452024-02-13T21:42:44.526Z 20 Acked 1019 Read 3 Sent Done Done false
85462024-02-13T21:42:44.526Z 21 Acked 1020 Write 6 Sent Done Done false
85472024-02-13T21:42:44.526Z 22 Acked 1021 Read 10 Sent Done Done false
85482024-02-13T21:42:44.526Z 23 Acked 1022 Read 3 Sent Done Done false
85492024-02-13T21:42:44.526Z 24 Acked 1023 Write 7 Sent Done Done false
85502024-02-13T21:42:44.526Z 25 Acked 1024 Write 6 Sent Done Done false
85512024-02-13T21:42:44.526Z 26 Acked 1025 Write 2 Sent Done Done false
85522024-02-13T21:42:44.526Z 27 Acked 1026 Read 6 Sent Done Done false
85532024-02-13T21:42:44.526Z 28 Acked 1027 Write 7 Sent Done Done false
85542024-02-13T21:42:44.526Z 29 Acked 1028 Write 2 Sent Sent Done false
85552024-02-13T21:42:44.526Z 30 Acked 1029 Write 2 Sent Sent Sent false
85562024-02-13T21:42:44.526Z STATES DS:0 DS:1 DS:2 TOTAL
85572024-02-13T21:42:44.526Z New 0 0 0 0
85582024-02-13T21:42:44.526Z Sent 30 2 1 33
85592024-02-13T21:42:44.526Z Done 0 28 29 57
85602024-02-13T21:42:44.526Z Skipped 0 0 0 0
85612024-02-13T21:42:44.526Z Error 0 0 0 0
85622024-02-13T21:42:44.527ZLast Flush: 0 1011 1011
85632024-02-13T21:42:44.527ZDownstairs last five completed:
85642024-02-13T21:42:44.527ZUpstairs last five completed: 30 29 28 27 26
85652024-02-13T21:42:44.527Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:42.474880126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1450}
85662024-02-13T21:42:44.527ZWrote out file "/tmp/test_repair_verify.data"
85672024-02-13T21:42:44.527Z
85682024-02-13T21:42:44.527ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85692024-02-13T21:42:44.527ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85702024-02-13T21:42:44.527Z 0 000-019 77 77 77 134 134 134 F F F
85712024-02-13T21:42:44.527Z 1 020-039 77 77 77 134 134 134 F F F
85722024-02-13T21:42:44.527Z 2 040-059 74 74 74 128 128 128 F T T <---
85732024-02-13T21:42:44.527Z 3 060-079 76 76 76 133 133 133 F T T <---
85742024-02-13T21:42:44.527Z 4 080-099 74 74 74 130 130 130 F T T <---
85752024-02-13T21:42:44.527Z 5 100-119 66 66 66 113 113 113 F T T <---
85762024-02-13T21:42:44.527Z 6 120-139 70 70 70 118 118 118 F F F
85772024-02-13T21:42:44.527Z 7 140-159 72 72 72 124 124 124 F F F
85782024-02-13T21:42:44.527Z 8 160-179 62 62 62 107 107 107 F F F
85792024-02-13T21:42:44.527Z 9 180-199 76 76 76 132 132 132 F F F
85802024-02-13T21:42:44.527Z 10 200-219 77 77 77 134 134 134 F F F
85812024-02-13T21:42:44.527Z 11 220-239 68 68 68 116 116 116 F F F
85822024-02-13T21:42:44.527Z 12 240-259 68 78 78 116 135 135 F T T <---
85832024-02-13T21:42:44.527Z 13 260-279 75 75 75 131 131 131 F T T <---
85842024-02-13T21:42:44.527Z 14 280-299 77 77 77 134 134 134 F T T <---
85852024-02-13T21:42:44.527Z 15 300-319 72 72 72 121 121 121 F F F
85862024-02-13T21:42:44.528Z 16 320-339 77 77 77 134 134 134 F T T <---
85872024-02-13T21:42:44.528Z 17 340-359 77 77 77 134 134 134 F T T <---
85882024-02-13T21:42:44.528Z 18 360-379 72 72 72 125 125 125 F T T <---
85892024-02-13T21:42:44.528Z 19 380-399 68 78 78 116 135 135 F T T <---
85902024-02-13T21:42:44.528Z 20 400-419 77 77 77 134 134 134 F T T <---
85912024-02-13T21:42:44.528Z 21 420-439 75 75 75 131 131 131 F T T <---
85922024-02-13T21:42:44.528Z 22 440-459 74 74 74 130 130 130 F T T <---
85932024-02-13T21:42:44.536Z 23 460-479 77 78 78 134 135 135 F F F <---
85942024-02-13T21:42:44.536Z 24 480-499 77 78 78 134 135 135 F F F <---
85952024-02-13T21:42:44.536Z 25 500-519 76 78 78 132 135 135 F F F <---
85962024-02-13T21:42:44.536Z 26 520-539 77 77 77 134 134 134 F F F
85972024-02-13T21:42:44.536Z 27 540-559 77 77 77 134 134 134 F T T <---
85982024-02-13T21:42:44.536Z 28 560-579 77 78 78 134 135 135 F F F <---
85992024-02-13T21:42:44.536Z 29 580-599 74 74 74 130 130 130 F F F
86002024-02-13T21:42:44.536ZMax gen: 78, Max flush: 135
86012024-02-13T21:42:44.536ZError: Difference in extent metadata found!
86022024-02-13T21:42:44.536ZA Difference in extent metadata is expected here
86032024-02-13T21:42:44.536ZOn loop 39
86042024-02-13T21:42:44.536Z
86052024-02-13T21:42:44.536ZVerifying data now
86062024-02-13T21:42:48.634ZNo change after verify
86072024-02-13T21:42:48.641ZLoop: 39 Downstairs dump after verify (and repair):
86082024-02-13T21:42:48.649ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86092024-02-13T21:42:48.649Z 0 000-019 77 77 77 134 134 134 F F F
86102024-02-13T21:42:48.649Z 1 020-039 77 77 77 134 134 134 F F F
86112024-02-13T21:42:48.649Z 2 040-059 79 79 79 136 136 136 F F F
86122024-02-13T21:42:48.649Z 3 060-079 79 79 79 136 136 136 F F F
86132024-02-13T21:42:48.649Z 4 080-099 79 79 79 136 136 136 F F F
86142024-02-13T21:42:48.649Z 5 100-119 79 79 79 136 136 136 F F F
86152024-02-13T21:42:48.649Z 6 120-139 70 70 70 118 118 118 F F F
86162024-02-13T21:42:48.649Z 7 140-159 72 72 72 124 124 124 F F F
86172024-02-13T21:42:48.649Z 8 160-179 62 62 62 107 107 107 F F F
86182024-02-13T21:42:48.649Z 9 180-199 76 76 76 132 132 132 F F F
86192024-02-13T21:42:48.655Z 10 200-219 77 77 77 134 134 134 F F F
86202024-02-13T21:42:48.655Z 11 220-239 68 68 68 116 116 116 F F F
86212024-02-13T21:42:48.655Z 12 240-259 79 79 79 136 136 136 F F F
86222024-02-13T21:42:48.655Z 13 260-279 79 79 79 136 136 136 F F F
86232024-02-13T21:42:48.655Z 14 280-299 79 79 79 136 136 136 F F F
86242024-02-13T21:42:48.655Z 15 300-319 72 72 72 121 121 121 F F F
86252024-02-13T21:42:48.655Z 16 320-339 79 79 79 136 136 136 F F F
86262024-02-13T21:42:48.655Z 17 340-359 79 79 79 136 136 136 F F F
86272024-02-13T21:42:48.655Z 18 360-379 79 79 79 136 136 136 F F F
86282024-02-13T21:42:48.655Z 19 380-399 79 79 79 136 136 136 F F F
86292024-02-13T21:42:48.656Z 20 400-419 79 79 79 136 136 136 F F F
86302024-02-13T21:42:48.656Z 21 420-439 79 79 79 136 136 136 F F F
86312024-02-13T21:42:48.656Z 22 440-459 79 79 79 136 136 136 F F F
86322024-02-13T21:42:48.656Z 23 460-479 78 78 78 135 135 135 F F F
86332024-02-13T21:42:48.656Z 24 480-499 78 78 78 135 135 135 F F F
86342024-02-13T21:42:48.656Z 25 500-519 78 78 78 135 135 135 F F F
86352024-02-13T21:42:48.656Z 26 520-539 77 77 77 134 134 134 F F F
86362024-02-13T21:42:48.656Z 27 540-559 79 79 79 136 136 136 F F F
86372024-02-13T21:42:48.656Z 28 560-579 78 78 78 135 135 135 F F F
86382024-02-13T21:42:48.656Z 29 580-599 74 74 74 130 130 130 F F F
86392024-02-13T21:42:48.656ZMax gen: 79, Max flush: 136
86402024-02-13T21:42:48.657Z
86412024-02-13T21:42:48.657ZBegin loop 40
86422024-02-13T21:42:48.657ZDownstairs to restart: 0
86432024-02-13T21:42:48.668Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443330201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
86442024-02-13T21:42:48.669Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\nCrucible runtime is spawned
86452024-02-13T21:42:48.675Z rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443675886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
86462024-02-13T21:42:48.675Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44371959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
86472024-02-13T21:42:48.675Z{"msg":"Crucible b6c110a1-7acf-40d4-8598-7d3607b66db2 has session id: 3727b13c-0c40-41a6-a129-127b8f43cd56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443738583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86482024-02-13T21:42:48.675Z{"msg":"Upstairs opts: Upstairs UUID: b6c110a1-7acf-40d4-8598-7d3607b66db2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443778188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86492024-02-13T21:42:48.675Z{"msg":"Crucible stats registered with UUID: b6c110a1-7acf-40d4-8598-7d3607b66db2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443795742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86502024-02-13T21:42:48.675Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443821263Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
86512024-02-13T21:42:48.675Z{"msg":"b6c110a1-7acf-40d4-8598-7d3607b66db2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443835998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86522024-02-13T21:42:48.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443882861Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86532024-02-13T21:42:48.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443910071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86542024-02-13T21:42:48.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443951126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86552024-02-13T21:42:48.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.443986333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86562024-02-13T21:42:48.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444057117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86572024-02-13T21:42:48.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44410617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86582024-02-13T21:42:48.676Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444211171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86592024-02-13T21:42:48.676Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444246179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86602024-02-13T21:42:48.676Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444276828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86612024-02-13T21:42:48.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444292502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86622024-02-13T21:42:48.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444306527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86632024-02-13T21:42:48.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444327639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"io task","client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86642024-02-13T21:42:48.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444823329Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86652024-02-13T21:42:48.676Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444870223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86662024-02-13T21:42:48.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.444915356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86672024-02-13T21:42:48.676Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44495994Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86682024-02-13T21:42:48.676Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445413355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86692024-02-13T21:42:48.676Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445533632Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86702024-02-13T21:42:48.676Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445835082Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86712024-02-13T21:42:48.676Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445882665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86722024-02-13T21:42:48.676Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445936256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86732024-02-13T21:42:48.676Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445955619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86742024-02-13T21:42:48.676Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445978001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86752024-02-13T21:42:48.676Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.445992296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86762024-02-13T21:42:48.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.446020525Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86772024-02-13T21:42:48.676Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.446036619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86782024-02-13T21:42:48.677Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.446545375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86792024-02-13T21:42:48.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.446949908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86802024-02-13T21:42:48.677Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.446985495Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86812024-02-13T21:42:48.677Z{"msg":"[0]R flush_numbers[0..12]: [134, 134, 136, 136, 136, 136, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447007407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86822024-02-13T21:42:48.677Z{"msg":"[0]R generation[0..12]: [77, 77, 79, 79, 79, 79, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447031768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86832024-02-13T21:42:48.677Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447050272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86842024-02-13T21:42:48.677Z{"msg":"[1]R flush_numbers[0..12]: [134, 134, 136, 136, 136, 136, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447065196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86852024-02-13T21:42:48.677Z{"msg":"[1]R generation[0..12]: [77, 77, 79, 79, 79, 79, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447079221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86862024-02-13T21:42:48.677Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447108021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86872024-02-13T21:42:48.677Z{"msg":"[2]R flush_numbers[0..12]: [134, 134, 136, 136, 136, 136, 118, 124, 107, 132, 134, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447123605Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86882024-02-13T21:42:48.677Z{"msg":"[2]R generation[0..12]: [77, 77, 79, 79, 79, 79, 70, 72, 62, 76, 77, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44713845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86892024-02-13T21:42:48.677Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447159982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}Wait for a query_work_queue command to finish before sending IO
86902024-02-13T21:42:48.677Z
86912024-02-13T21:42:48.677Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447182714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86922024-02-13T21:42:48.677Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447199418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86932024-02-13T21:42:48.677Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447222569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86942024-02-13T21:42:48.677Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447254627Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"":"downstairs","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86952024-02-13T21:42:48.677Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44727482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86962024-02-13T21:42:48.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447306679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"0","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86972024-02-13T21:42:48.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447326971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"1","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86982024-02-13T21:42:48.677Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44735966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"client":"2","":"downstairs","session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
86992024-02-13T21:42:48.677Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44738583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
87002024-02-13T21:42:48.677Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447409042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
87012024-02-13T21:42:48.677Z{"msg":"b6c110a1-7acf-40d4-8598-7d3607b66db2 is now active with session: 3727b13c-0c40-41a6-a129-127b8f43cd56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447422627Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
87022024-02-13T21:42:48.677Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.44744105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458,"session_id":"3727b13c-0c40-41a6-a129-127b8f43cd56"}
87032024-02-13T21:42:48.678Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:47.447453615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
87042024-02-13T21:42:48.678ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
87052024-02-13T21:42:48.682ZLoading write count information from file "/tmp/test_repair_verify.data"
87062024-02-13T21:42:48.682ZRun Repair workload
87072024-02-13T21:42:48.682Z01/30 Write block 371 len 16384 data: 8 8 8 7
87082024-02-13T21:42:48.682Z02/30 Write block 138 len 32768 data: 8 6 6 7 9 7 7 7
87092024-02-13T21:42:48.682Z03/30 Read block 45 len 28672
87102024-02-13T21:42:48.682Z04/30 Write block 211 len 4096 data: 7
87112024-02-13T21:42:48.682Z05/30 Write block 562 len 36864 data: 4 4 7 8 8 9 10 8 6
87122024-02-13T21:42:48.682Z06/30 Write block 201 len 4096 data: 9
87132024-02-13T21:42:48.682Z07/30 Read block 457 len 4096
87142024-02-13T21:42:48.682Z08/30 Read block 536 len 4096
87152024-02-13T21:42:48.682Z09/30 Read block 442 len 4096
87162024-02-13T21:42:48.682Z10/30 Read block 224 len 16384
87172024-02-13T21:42:48.682Z11/30 Flush
87182024-02-13T21:42:48.686Z12/30 Write block 139 len 40960 data: 7 7 8 10 8 8 8 6 6 5
87192024-02-13T21:42:48.686Z13/30 Read block 353 len 40960
87202024-02-13T21:42:48.689Z14/30 Read block 422 len 24576
87212024-02-13T21:42:48.689Z15/30 Read block 408 len 28672
87222024-02-13T21:42:48.689Z16/30 Write block 434 len 32768 data: 10 10 6 6 7 7 6 7
87232024-02-13T21:42:48.689Z17/30 Read block 402 len 4096
87242024-02-13T21:42:48.692Z18/30 Read block 168 len 20480
87252024-02-13T21:42:48.692Z19/30 Write block 321 len 36864 data: 7 7 9 8 10 10 10 8 6
87262024-02-13T21:42:48.692Z20/30 Read block 539 len 40960
87272024-02-13T21:42:48.695Z21/30 Write block 227 len 4096 data: 6
87282024-02-13T21:42:48.695Z22/30 Read block 394 len 8192
87292024-02-13T21:42:48.695Z23/30 Flush
87302024-02-13T21:42:48.701Z24/30 Write block 25 len 28672 data: 7 6 5 5 5 4 7
87312024-02-13T21:42:48.702Z25/30 Write block 376 len 8192 data: 7 7
87322024-02-13T21:42:48.702Z26/30 Read block 117 len 20480
87332024-02-13T21:42:48.705Z27/30 Write block 355 len 36864 data: 8 10 10 9 9 8 5 5 5
87342024-02-13T21:42:48.705Z28/30 Read block 186 len 16384
87352024-02-13T21:42:48.705Z29/30 Read block 342 len 16384
87362024-02-13T21:42:48.705Z30/30 Read block 307 len 28672
87372024-02-13T21:42:48.705Z----------------------------------------------------------------
87382024-02-13T21:42:48.705Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87392024-02-13T21:42:48.705ZStates: Active Active Active
87402024-02-13T21:42:48.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87412024-02-13T21:42:48.705Z 1 Acked 1000 Write 4 Sent Done Done false
87422024-02-13T21:42:48.705Z 2 Acked 1001 Write 8 Sent Done Done false
87432024-02-13T21:42:48.705Z 3 Acked 1002 Read 7 Sent Done Done false
87442024-02-13T21:42:48.706Z 4 Acked 1003 Write 1 Sent Done Done false
87452024-02-13T21:42:48.706Z 5 Acked 1004 Write 9 Sent Done Done false
87462024-02-13T21:42:48.706Z 6 Acked 1005 Write 1 Sent Done Done false
87472024-02-13T21:42:48.706Z 7 Acked 1006 Read 1 Sent Done Done false
87482024-02-13T21:42:48.706Z 8 Acked 1007 Read 1 Sent Done Done false
87492024-02-13T21:42:48.706Z 9 Acked 1008 Read 1 Sent Done Done false
87502024-02-13T21:42:48.706Z 10 Acked 1009 Read 4 Sent Done Done false
87512024-02-13T21:42:48.706Z 11 Acked 1010 Flush 0 Sent Done Done false
87522024-02-13T21:42:48.706Z 12 Acked 1011 Write 10 Sent Done Done false
87532024-02-13T21:42:48.706Z 13 Acked 1012 Read 10 Sent Done Done false
87542024-02-13T21:42:48.706Z 14 Acked 1013 Read 6 Sent Done Done false
87552024-02-13T21:42:48.706Z 15 Acked 1014 Read 7 Sent Done Done false
87562024-02-13T21:42:48.706Z 16 Acked 1015 Write 8 Sent Done Done false
87572024-02-13T21:42:48.706Z 17 Acked 1016 Read 1 Sent Done Done false
87582024-02-13T21:42:48.706Z 18 Acked 1017 Read 5 Sent Done Done false
87592024-02-13T21:42:48.706Z 19 Acked 1018 Write 9 Sent Done Done false
87602024-02-13T21:42:48.706Z 20 Acked 1019 Read 10 Sent Done Done false
87612024-02-13T21:42:48.706Z 21 Acked 1020 Write 1 Sent Done Done false
87622024-02-13T21:42:48.706Z 22 Acked 1021 Read 2 Sent Done Done false
87632024-02-13T21:42:48.706Z 23 Acked 1022 Flush 0 Sent Done Done false
87642024-02-13T21:42:48.706Z 24 Acked 1023 Write 7 Sent Done Done false
87652024-02-13T21:42:48.706Z 25 Acked 1024 Write 2 Sent Done Done false
87662024-02-13T21:42:48.706Z 26 Acked 1025 Read 5 Sent Done Done false
87672024-02-13T21:42:48.706Z 27 Acked 1026 Write 9 Sent Done Done false
87682024-02-13T21:42:48.707Z 28 Acked 1027 Read 4 Sent Done Done false
87692024-02-13T21:42:48.707Z 29 Acked 1028 Read 4 Sent Done Done false
87702024-02-13T21:42:48.707Z 30 Acked 1029 Read 7 Sent Done Done false
87712024-02-13T21:42:48.707Z STATES DS:0 DS:1 DS:2 TOTAL
87722024-02-13T21:42:48.707Z New 0 0 0 0
87732024-02-13T21:42:48.707Z Sent 30 0 0 30
87742024-02-13T21:42:48.707Z Done 0 30 30 60
87752024-02-13T21:42:48.707Z Skipped 0 0 0 0
87762024-02-13T21:42:48.707Z Error 0 0 0 0
87772024-02-13T21:42:48.707ZLast Flush: 0 1022 1022
87782024-02-13T21:42:48.707ZDownstairs last five completed:
87792024-02-13T21:42:48.707ZUpstairs last five completed: 30 29 28 27 26
87802024-02-13T21:42:48.707Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:47.478595769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1458}
87812024-02-13T21:42:48.707ZWrote out file "/tmp/test_repair_verify.data"
87822024-02-13T21:42:48.710Z
87832024-02-13T21:42:48.710ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87842024-02-13T21:42:48.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87852024-02-13T21:42:48.725Z 0 000-019 77 77 77 134 134 134 F F F
87862024-02-13T21:42:48.726Z 1 020-039 77 77 77 134 134 134 F T T <---
87872024-02-13T21:42:48.726Z 2 040-059 79 79 79 136 136 136 F F F
87882024-02-13T21:42:48.726Z 3 060-079 79 79 79 136 136 136 F F F
87892024-02-13T21:42:48.726Z 4 080-099 79 79 79 136 136 136 F F F
87902024-02-13T21:42:48.726Z 5 100-119 79 79 79 136 136 136 F F F
87912024-02-13T21:42:48.726Z 6 120-139 70 80 80 118 138 138 F F F <---
87922024-02-13T21:42:48.726Z 7 140-159 72 80 80 124 138 138 F F F <---
87932024-02-13T21:42:48.726Z 8 160-179 62 62 62 107 107 107 F F F
87942024-02-13T21:42:48.729Z 9 180-199 76 76 76 132 132 132 F F F
87952024-02-13T21:42:48.729Z 10 200-219 77 80 80 134 137 137 F F F <---
87962024-02-13T21:42:48.729Z 11 220-239 68 80 80 116 138 138 F F F <---
87972024-02-13T21:42:48.729Z 12 240-259 79 79 79 136 136 136 F F F
87982024-02-13T21:42:48.729Z 13 260-279 79 79 79 136 136 136 F F F
87992024-02-13T21:42:48.729Z 14 280-299 79 79 79 136 136 136 F F F
88002024-02-13T21:42:48.729Z 15 300-319 72 72 72 121 121 121 F F F
88012024-02-13T21:42:48.729Z 16 320-339 79 80 80 136 138 138 F F F <---
88022024-02-13T21:42:48.730Z 17 340-359 79 79 79 136 136 136 F T T <---
88032024-02-13T21:42:48.730Z 18 360-379 79 80 80 136 137 137 F T T <---
88042024-02-13T21:42:48.730Z 19 380-399 79 79 79 136 136 136 F F F
88052024-02-13T21:42:48.730Z 20 400-419 79 79 79 136 136 136 F F F
88062024-02-13T21:42:48.730Z 21 420-439 79 80 80 136 138 138 F F F <---
88072024-02-13T21:42:48.730Z 22 440-459 79 80 80 136 138 138 F F F <---
88082024-02-13T21:42:48.730Z 23 460-479 78 78 78 135 135 135 F F F
88092024-02-13T21:42:48.730Z 24 480-499 78 78 78 135 135 135 F F F
88102024-02-13T21:42:48.730Z 25 500-519 78 78 78 135 135 135 F F F
88112024-02-13T21:42:48.730Z 26 520-539 77 77 77 134 134 134 F F F
88122024-02-13T21:42:48.730Z 27 540-559 79 79 79 136 136 136 F F F
88132024-02-13T21:42:48.730Z 28 560-579 78 80 80 135 137 137 F F F <---
88142024-02-13T21:42:48.730Z 29 580-599 74 74 74 130 130 130 F F F
88152024-02-13T21:42:48.730ZMax gen: 80, Max flush: 138
88162024-02-13T21:42:48.730ZError: Difference in extent metadata found!
88172024-02-13T21:42:48.730ZA Difference in extent metadata is expected here
88182024-02-13T21:42:48.730ZOn loop 40
88192024-02-13T21:42:48.730Z
88202024-02-13T21:42:48.733ZVerifying data now
88212024-02-13T21:42:53.291ZNo change after verify
88222024-02-13T21:42:53.291ZLoop: 40 Downstairs dump after verify (and repair):
88232024-02-13T21:42:53.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88242024-02-13T21:42:53.305Z 0 000-019 77 77 77 134 134 134 F F F
88252024-02-13T21:42:53.305Z 1 020-039 81 81 81 139 139 139 F F F
88262024-02-13T21:42:53.305Z 2 040-059 79 79 79 136 136 136 F F F
88272024-02-13T21:42:53.305Z 3 060-079 79 79 79 136 136 136 F F F
88282024-02-13T21:42:53.305Z 4 080-099 79 79 79 136 136 136 F F F
88292024-02-13T21:42:53.305Z 5 100-119 79 79 79 136 136 136 F F F
88302024-02-13T21:42:53.305Z 6 120-139 80 80 80 138 138 138 F F F
88312024-02-13T21:42:53.305Z 7 140-159 80 80 80 138 138 138 F F F
88322024-02-13T21:42:53.305Z 8 160-179 62 62 62 107 107 107 F F F
88332024-02-13T21:42:53.306Z 9 180-199 76 76 76 132 132 132 F F F
88342024-02-13T21:42:53.306Z 10 200-219 80 80 80 137 137 137 F F F
88352024-02-13T21:42:53.309Z 11 220-239 80 80 80 138 138 138 F F F
88362024-02-13T21:42:53.309Z 12 240-259 79 79 79 136 136 136 F F F
88372024-02-13T21:42:53.309Z 13 260-279 79 79 79 136 136 136 F F F
88382024-02-13T21:42:53.309Z 14 280-299 79 79 79 136 136 136 F F F
88392024-02-13T21:42:53.309Z 15 300-319 72 72 72 121 121 121 F F F
88402024-02-13T21:42:53.309Z 16 320-339 80 80 80 138 138 138 F F F
88412024-02-13T21:42:53.309Z 17 340-359 81 81 81 139 139 139 F F F
88422024-02-13T21:42:53.309Z 18 360-379 81 81 81 139 139 139 F F F
88432024-02-13T21:42:53.309Z 19 380-399 79 79 79 136 136 136 F F F
88442024-02-13T21:42:53.309Z 20 400-419 79 79 79 136 136 136 F F F
88452024-02-13T21:42:53.309Z 21 420-439 80 80 80 138 138 138 F F F
88462024-02-13T21:42:53.309Z 22 440-459 80 80 80 138 138 138 F F F
88472024-02-13T21:42:53.309Z 23 460-479 78 78 78 135 135 135 F F F
88482024-02-13T21:42:53.309Z 24 480-499 78 78 78 135 135 135 F F F
88492024-02-13T21:42:53.310Z 25 500-519 78 78 78 135 135 135 F F F
88502024-02-13T21:42:53.310Z 26 520-539 77 77 77 134 134 134 F F F
88512024-02-13T21:42:53.310Z 27 540-559 79 79 79 136 136 136 F F F
88522024-02-13T21:42:53.310Z 28 560-579 80 80 80 137 137 137 F F F
88532024-02-13T21:42:53.310Z 29 580-599 74 74 74 130 130 130 F F F
88542024-02-13T21:42:53.310ZMax gen: 81, Max flush: 139
88552024-02-13T21:42:53.310Z
88562024-02-13T21:42:53.310ZBegin loop 41
88572024-02-13T21:42:53.310ZDownstairs to restart: 1
88582024-02-13T21:42:53.326Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.099857151Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
88592024-02-13T21:42:53.331Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100147415Z","Crucible runtime is spawned
88602024-02-13T21:42:53.331Zhostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
88612024-02-13T21:42:53.332Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100191539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
88622024-02-13T21:42:53.332Z{"msg":"Crucible 0106b2be-1af0-49f2-9083-73e301b46c33 has session id: 2ab23317-d3b4-4c0a-85b3-8ff132628981","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100234604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88632024-02-13T21:42:53.332Z{"msg":"Upstairs opts: Upstairs UUID: 0106b2be-1af0-49f2-9083-73e301b46c33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100281237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88642024-02-13T21:42:53.332Z{"msg":"Crucible stats registered with UUID: 0106b2be-1af0-49f2-9083-73e301b46c33","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100319723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88652024-02-13T21:42:53.332Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100347693Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
88662024-02-13T21:42:53.332Z{"msg":"0106b2be-1af0-49f2-9083-73e301b46c33 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100385639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88672024-02-13T21:42:53.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100413929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88682024-02-13T21:42:53.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100443148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88692024-02-13T21:42:53.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100471898Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88702024-02-13T21:42:53.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100519111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88712024-02-13T21:42:53.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100584497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88722024-02-13T21:42:53.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.10063187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88732024-02-13T21:42:53.332Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.10065849Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88742024-02-13T21:42:53.332Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100705043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88752024-02-13T21:42:53.332Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100755725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88762024-02-13T21:42:53.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100776547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88772024-02-13T21:42:53.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.10079727Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88782024-02-13T21:42:53.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.100814313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"io task","client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88792024-02-13T21:42:53.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.101498685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88802024-02-13T21:42:53.332Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.101544489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88812024-02-13T21:42:53.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.101575827Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88822024-02-13T21:42:53.332Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.101616942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88832024-02-13T21:42:53.332Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102109803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88842024-02-13T21:42:53.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102248243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88852024-02-13T21:42:53.333Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102288039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88862024-02-13T21:42:53.333Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102318598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88872024-02-13T21:42:53.333Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102576914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88882024-02-13T21:42:53.333Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102631434Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88892024-02-13T21:42:53.333Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102678817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88902024-02-13T21:42:53.333Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102787028Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88912024-02-13T21:42:53.333Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102826164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88922024-02-13T21:42:53.333Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102851625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88932024-02-13T21:42:53.333Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.102880464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88942024-02-13T21:42:53.338Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103282298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88952024-02-13T21:42:53.338Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103324733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88962024-02-13T21:42:53.338Z{"msg":"[0]R flush_numbers[0..12]: [134, 139, 136, 136, 136, 136, 138, 138, 107, 132, 137, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103342696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88972024-02-13T21:42:53.338Z{"msg":"[0]R generation[0..12]: [77, 81, 79, 79, 79, 79, 80, 80, 62, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103357551Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88982024-02-13T21:42:53.338Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103380293Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
88992024-02-13T21:42:53.338Z{"msg":"[1]R flush_numbers[0..12]: [134, 139, 136, 136, 136, 136, 138, 138, 107, 132, 137, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103395147Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89002024-02-13T21:42:53.338Z{"msg":"[1]R generation[0..12]: [77, 81, 79, 79, 79, 79, 80, 80, 62, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.1034165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89012024-02-13T21:42:53.338Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103432294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89022024-02-13T21:42:53.338Z{"msg":"[2]R flush_numbers[0..12]: [134, 139, 136, 136, 136, 136, 138, 138, 107, 132, 137, 138]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO
89032024-02-13T21:42:53.338Z"level":30,"time":"2024-02-13T21:42:52.103465132Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89042024-02-13T21:42:53.338Z{"msg":"[2]R generation[0..12]: [77, 81, 79, 79, 79, 79, 80, 80, 62, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103480776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89052024-02-13T21:42:53.338Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103495021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89062024-02-13T21:42:53.338Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103509226Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89072024-02-13T21:42:53.338Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103529528Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89082024-02-13T21:42:53.338Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103542714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89092024-02-13T21:42:53.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103562067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"":"downstairs","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89102024-02-13T21:42:53.338Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103575702Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89112024-02-13T21:42:53.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103594695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"0","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89122024-02-13T21:42:53.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.10360817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"1","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89132024-02-13T21:42:53.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103628073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"client":"2","":"downstairs","session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89142024-02-13T21:42:53.338Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103641708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89152024-02-13T21:42:53.339Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103660501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89162024-02-13T21:42:53.339Z{"msg":"0106b2be-1af0-49f2-9083-73e301b46c33 is now active with session: 2ab23317-d3b4-4c0a-85b3-8ff132628981","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103673756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89172024-02-13T21:42:53.339Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.10369195Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466,"session_id":"2ab23317-d3b4-4c0a-85b3-8ff132628981"}
89182024-02-13T21:42:53.339Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:52.103704325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
89192024-02-13T21:42:53.339ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89202024-02-13T21:42:53.339ZLoading write count information from file "/tmp/test_repair_verify.data"
89212024-02-13T21:42:53.339ZRun Repair workload
89222024-02-13T21:42:53.339Z01/30 Flush
89232024-02-13T21:42:53.339Z02/30 Write block 411 len 12288 data: 8 5 6
89242024-02-13T21:42:53.339Z03/30 Flush
89252024-02-13T21:42:53.339Z04/30 Write block 259 len 4096 data: 4
89262024-02-13T21:42:53.339Z05/30 Write block 142 len 4096 data: 11
89272024-02-13T21:42:53.339Z06/30 Write block 180 len 28672 data: 5 4 4 7 7 10 9
89282024-02-13T21:42:53.342Z07/30 Read block 531 len 20480
89292024-02-13T21:42:53.342Z08/30 Flush
89302024-02-13T21:42:53.342Z09/30 Write block 276 len 40960 data: 10 12 10 10 9 5 4 4 4 5
89312024-02-13T21:42:53.342Z10/30 Read block 440 len 16384
89322024-02-13T21:42:53.348Z11/30 Read block 176 len 4096
89332024-02-13T21:42:53.348Z12/30 Write block 288 len 32768 data: 6 7 7 7 8 10 10 8
89342024-02-13T21:42:53.348Z13/30 Write block 220 len 16384 data: 7 7 8 8
89352024-02-13T21:42:53.348Z14/30 Flush
89362024-02-13T21:42:53.351Z15/30 Write block 457 len 16384 data: 6 7 9 9
89372024-02-13T21:42:53.352Z16/30 Flush
89382024-02-13T21:42:53.352Z17/30 Write block 487 len 32768 data: 6 6 6 4 5 5 7 10
89392024-02-13T21:42:53.352Z18/30 Read block 117 len 8192
89402024-02-13T21:42:53.355Z19/30 Write block 90 len 20480 data: 4 4 4 4 4
89412024-02-13T21:42:53.355Z20/30 Read block 413 len 12288
89422024-02-13T21:42:53.355Z21/30 Read block 53 len 12288
89432024-02-13T21:42:53.355Z22/30 Read block 482 len 40960
89442024-02-13T21:42:53.358Z23/30 Flush
89452024-02-13T21:42:53.361Z24/30 Flush
89462024-02-13T21:42:53.361Z25/30 Write block 91 len 16384 data: 5 5 5 5
89472024-02-13T21:42:53.361Z26/30 Read block 61 len 36864
89482024-02-13T21:42:53.361Z27/30 Read block 455 len 20480
89492024-02-13T21:42:53.365Z28/30 Write block 376 len 4096 data: 8
89502024-02-13T21:42:53.365Z29/30 Write block 149 len 8192 data: 5 5
89512024-02-13T21:42:53.365Z30/30 Read block 546 len 28672
89522024-02-13T21:42:53.365Z----------------------------------------------------------------
89532024-02-13T21:42:53.365Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89542024-02-13T21:42:53.365ZStates: Active Active Active
89552024-02-13T21:42:53.365ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89562024-02-13T21:42:53.365Z 1 Acked 1000 Flush 0 Done Sent Done false
89572024-02-13T21:42:53.366Z 2 Acked 1001 Write 3 Done Sent Done false
89582024-02-13T21:42:53.366Z 3 Acked 1002 Flush 0 Done Sent Done false
89592024-02-13T21:42:53.366Z 4 Acked 1003 Write 1 Done Sent Done false
89602024-02-13T21:42:53.366Z 5 Acked 1004 Write 1 Done Sent Done false
89612024-02-13T21:42:53.366Z 6 Acked 1005 Write 7 Done Sent Done false
89622024-02-13T21:42:53.366Z 7 Acked 1006 Read 5 Done Sent Done false
89632024-02-13T21:42:53.366Z 8 Acked 1007 Flush 0 Done Sent Done false
89642024-02-13T21:42:53.366Z 9 Acked 1008 Write 10 Done Sent Done false
89652024-02-13T21:42:53.366Z 10 Acked 1009 Read 4 Done Sent Done false
89662024-02-13T21:42:53.366Z 11 Acked 1010 Read 1 Done Sent Done false
89672024-02-13T21:42:53.366Z 12 Acked 1011 Write 8 Done Sent Done false
89682024-02-13T21:42:53.366Z 13 Acked 1012 Write 4 Done Sent Done false
89692024-02-13T21:42:53.366Z 14 Acked 1013 Flush 0 Done Sent Done false
89702024-02-13T21:42:53.366Z 15 Acked 1014 Write 4 Done Sent Done false
89712024-02-13T21:42:53.366Z 16 Acked 1015 Flush 0 Done Sent Done false
89722024-02-13T21:42:53.366Z 17 Acked 1016 Write 8 Done Sent Done false
89732024-02-13T21:42:53.366Z 18 Acked 1017 Read 2 Done Sent Done false
89742024-02-13T21:42:53.366Z 19 Acked 1018 Write 5 Done Sent Done false
89752024-02-13T21:42:53.366Z 20 Acked 1019 Read 3 Done Sent Done false
89762024-02-13T21:42:53.366Z 21 Acked 1020 Read 3 Done Sent Done false
89772024-02-13T21:42:53.366Z 22 Acked 1021 Read 10 Done Sent Done false
89782024-02-13T21:42:53.366Z 23 Acked 1022 Flush 0 Done Sent Done false
89792024-02-13T21:42:53.366Z 24 Acked 1023 Flush 0 Done Sent Done false
89802024-02-13T21:42:53.366Z 25 Acked 1024 Write 4 Done Sent Done false
89812024-02-13T21:42:53.366Z 26 Acked 1025 Read 9 Done Sent Done false
89822024-02-13T21:42:53.367Z 27 Acked 1026 Read 5 Done Sent Done false
89832024-02-13T21:42:53.367Z 28 Acked 1027 Write 1 Done Sent Done false
89842024-02-13T21:42:53.367Z 29 Acked 1028 Write 2 Done Sent Done false
89852024-02-13T21:42:53.367Z 30 Acked 1029 Read 7 Done Sent Done false
89862024-02-13T21:42:53.367Z STATES DS:0 DS:1 DS:2 TOTAL
89872024-02-13T21:42:53.367Z New 0 0 0 0
89882024-02-13T21:42:53.367Z Sent 0 30 0 30
89892024-02-13T21:42:53.367Z Done 30 0 30 60
89902024-02-13T21:42:53.367Z Skipped 0 0 0 0
89912024-02-13T21:42:53.367Z Error 0 0 0 0
89922024-02-13T21:42:53.367ZLast Flush: 1023 0 1023
89932024-02-13T21:42:53.367ZDownstairs last five completed:
89942024-02-13T21:42:53.367ZUpstairs last five completed: 30 29 28 27 26
89952024-02-13T21:42:53.367Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:52.13740657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1466}
89962024-02-13T21:42:53.367ZWrote out file "/tmp/test_repair_verify.data"
89972024-02-13T21:42:53.370Z
89982024-02-13T21:42:53.370ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89992024-02-13T21:42:53.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90002024-02-13T21:42:53.390Z 0 000-019 77 77 77 134 134 134 F F F
90012024-02-13T21:42:53.390Z 1 020-039 81 81 81 139 139 139 F F F
90022024-02-13T21:42:53.391Z 2 040-059 79 79 79 136 136 136 F F F
90032024-02-13T21:42:53.391Z 3 060-079 79 79 79 136 136 136 F F F
90042024-02-13T21:42:53.822Z 4 080-099 82 79 82 145 136 145 T F T <---
90052024-02-13T21:42:53.822Z 5 100-119 79 79 79 136 136 136 F F F
90062024-02-13T21:42:53.822Z 6 120-139 80 80 80 138 138 138 F F F
90072024-02-13T21:42:53.822Z 7 140-159 82 80 82 142 138 142 T F T <---
90082024-02-13T21:42:53.822Z 8 160-179 62 62 62 107 107 107 F F F
90092024-02-13T21:42:53.822Z 9 180-199 82 76 82 142 132 142 F F F <---
90102024-02-13T21:42:53.822Z 10 200-219 80 80 80 137 137 137 F F F
90112024-02-13T21:42:53.822Z 11 220-239 82 80 82 143 138 143 F F F <---
90122024-02-13T21:42:53.822Z 12 240-259 82 79 82 142 136 142 F F F <---
90132024-02-13T21:42:53.822Z 13 260-279 82 79 82 143 136 143 F F F <---
90142024-02-13T21:42:53.822Z 14 280-299 82 79 82 143 136 143 F F F <---
90152024-02-13T21:42:53.822Z 15 300-319 72 72 72 121 121 121 F F F
90162024-02-13T21:42:53.823Z 16 320-339 80 80 80 138 138 138 F F F
90172024-02-13T21:42:53.823Z 17 340-359 81 81 81 139 139 139 F F F
90182024-02-13T21:42:53.823Z 18 360-379 81 81 81 139 139 139 T F T <---
90192024-02-13T21:42:53.823Z 19 380-399 79 79 79 136 136 136 F F F
90202024-02-13T21:42:53.823Z 20 400-419 82 79 82 141 136 141 F F F <---
90212024-02-13T21:42:53.823Z 21 420-439 80 80 80 138 138 138 F F F
90222024-02-13T21:42:53.823Z 22 440-459 82 80 82 144 138 144 F F F <---
90232024-02-13T21:42:53.823Z 23 460-479 82 78 82 144 135 144 F F F <---
90242024-02-13T21:42:53.823Z 24 480-499 82 78 82 145 135 145 F F F <---
90252024-02-13T21:42:53.823Z 25 500-519 78 78 78 135 135 135 F F F
90262024-02-13T21:42:53.823Z 26 520-539 77 77 77 134 134 134 F F F
90272024-02-13T21:42:53.823Z 27 540-559 79 79 79 136 136 136 F F F
90282024-02-13T21:42:53.823Z 28 560-579 80 80 80 137 137 137 F F F
90292024-02-13T21:42:53.823Z 29 580-599 74 74 74 130 130 130 F F F
90302024-02-13T21:42:53.823ZMax gen: 82, Max flush: 145
90312024-02-13T21:42:53.823ZError: Difference in extent metadata found!
90322024-02-13T21:42:53.823ZA Difference in extent metadata is expected here
90332024-02-13T21:42:53.823ZOn loop 41
90342024-02-13T21:42:53.823Z
90352024-02-13T21:42:53.823ZVerifying data now
90362024-02-13T21:42:57.991ZNo change after verify
90372024-02-13T21:42:57.991ZLoop: 41 Downstairs dump after verify (and repair):
90382024-02-13T21:42:58.005ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90392024-02-13T21:42:58.005Z 0 000-019 77 77 77 134 134 134 F F F
90402024-02-13T21:42:58.005Z 1 020-039 81 81 81 139 139 139 F F F
90412024-02-13T21:42:58.005Z 2 040-059 79 79 79 136 136 136 F F F
90422024-02-13T21:42:58.005Z 3 060-079 79 79 79 136 136 136 F F F
90432024-02-13T21:42:58.005Z 4 080-099 83 83 83 146 146 146 F F F
90442024-02-13T21:42:58.006Z 5 100-119 79 79 79 136 136 136 F F F
90452024-02-13T21:42:58.006Z 6 120-139 80 80 80 138 138 138 F F F
90462024-02-13T21:42:58.006Z 7 140-159 83 83 83 146 146 146 F F F
90472024-02-13T21:42:58.006Z 8 160-179 62 62 62 107 107 107 F F F
90482024-02-13T21:42:58.006Z 9 180-199 82 82 82 142 142 142 F F F
90492024-02-13T21:42:58.006Z 10 200-219 80 80 80 137 137 137 F F F
90502024-02-13T21:42:58.010Z 11 220-239 82 82 82 143 143 143 F F F
90512024-02-13T21:42:58.010Z 12 240-259 82 82 82 142 142 142 F F F
90522024-02-13T21:42:58.010Z 13 260-279 82 82 82 143 143 143 F F F
90532024-02-13T21:42:58.011Z 14 280-299 82 82 82 143 143 143 F F F
90542024-02-13T21:42:58.011Z 15 300-319 72 72 72 121 121 121 F F F
90552024-02-13T21:42:58.011Z 16 320-339 80 80 80 138 138 138 F F F
90562024-02-13T21:42:58.011Z 17 340-359 81 81 81 139 139 139 F F F
90572024-02-13T21:42:58.011Z 18 360-379 83 83 83 146 146 146 F F F
90582024-02-13T21:42:58.011Z 19 380-399 79 79 79 136 136 136 F F F
90592024-02-13T21:42:58.011Z 20 400-419 82 82 82 141 141 141 F F F
90602024-02-13T21:42:58.011Z 21 420-439 80 80 80 138 138 138 F F F
90612024-02-13T21:42:58.011Z 22 440-459 82 82 82 144 144 144 F F F
90622024-02-13T21:42:58.011Z 23 460-479 82 82 82 144 144 144 F F F
90632024-02-13T21:42:58.011Z 24 480-499 82 82 82 145 145 145 F F F
90642024-02-13T21:42:58.012Z 25 500-519 78 78 78 135 135 135 F F F
90652024-02-13T21:42:58.012Z 26 520-539 77 77 77 134 134 134 F F F
90662024-02-13T21:42:58.012Z 27 540-559 79 79 79 136 136 136 F F F
90672024-02-13T21:42:58.012Z 28 560-579 80 80 80 137 137 137 F F F
90682024-02-13T21:42:58.012Z 29 580-599 74 74 74 130 130 130 F F F
90692024-02-13T21:42:58.012ZMax gen: 83, Max flush: 146
90702024-02-13T21:42:58.012Z
90712024-02-13T21:42:58.012ZBegin loop 42
90722024-02-13T21:42:58.012ZDownstairs to restart: 1
90732024-02-13T21:42:58.026Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.796917478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
90742024-02-13T21:42:58.026Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797184911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
90752024-02-13T21:42:58.033Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797231924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
90762024-02-13T21:42:58.033ZCrucible runtime is spawned
90772024-02-13T21:42:58.033Z{"msg":"Crucible b652d5b2-b55e-44f1-b573-ac92f25d8d72 has session id: b04a3a8f-a3a4-408b-a278-e18df6c9a54c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797286324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90782024-02-13T21:42:58.033Z{"msg":"Upstairs opts: Upstairs UUID: b652d5b2-b55e-44f1-b573-ac92f25d8d72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797341664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90792024-02-13T21:42:58.033Z{"msg":"Crucible stats registered with UUID: b652d5b2-b55e-44f1-b573-ac92f25d8d72","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797371243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90802024-02-13T21:42:58.033Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797423924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
90812024-02-13T21:42:58.033Z{"msg":"b652d5b2-b55e-44f1-b573-ac92f25d8d72 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797465569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90822024-02-13T21:42:58.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797507864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90832024-02-13T21:42:58.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797540172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90842024-02-13T21:42:58.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797583116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90852024-02-13T21:42:58.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797657569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90862024-02-13T21:42:58.034Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797713759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90872024-02-13T21:42:58.034Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.79773947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90882024-02-13T21:42:58.034Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797762561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90892024-02-13T21:42:58.035Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797789431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90902024-02-13T21:42:58.035Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797811494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90912024-02-13T21:42:58.035Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797833586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90922024-02-13T21:42:58.035Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797855687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90932024-02-13T21:42:58.035Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.797879219Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"io task","client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90942024-02-13T21:42:58.035Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.798567049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90952024-02-13T21:42:58.035Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.798622779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90962024-02-13T21:42:58.035Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.798650449Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90972024-02-13T21:42:58.036Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.798683237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90982024-02-13T21:42:58.036Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799256519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
90992024-02-13T21:42:58.036Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799339459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91002024-02-13T21:42:58.036Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.79980593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91012024-02-13T21:42:58.036Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799853033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91022024-02-13T21:42:58.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799876015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91032024-02-13T21:42:58.037Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799905584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91042024-02-13T21:42:58.037Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799920769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91052024-02-13T21:42:58.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799935533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91062024-02-13T21:42:58.037Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.79997285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91072024-02-13T21:42:58.037Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.799989114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91082024-02-13T21:42:58.037Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.800454145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91092024-02-13T21:42:58.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.800841255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91102024-02-13T21:42:58.037Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.800883589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91112024-02-13T21:42:58.038Z{"msg":"[0]R flush_numbers[0..12]: [134, 139, 136, 136, 146, 136, 138, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.800910759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91122024-02-13T21:42:58.038Z{"msg":"[0]R generation[0..12]: [77, 81, 79, 79, 83, 79, 80, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.800939069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91132024-02-13T21:42:58.038Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.80096559Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91142024-02-13T21:42:58.038Z{"msg":"[1]R flush_numbers[0..12]: [134, 139, 136, 136, 146, 136, 138, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
91152024-02-13T21:42:58.038Z,"time":"2024-02-13T21:42:56.80099139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91162024-02-13T21:42:58.038Z{"msg":"[1]R generation[0..12]: [77, 81, 79, 79, 83, 79, 80, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801031975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91172024-02-13T21:42:58.038Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.80104719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91182024-02-13T21:42:58.039Z{"msg":"[2]R flush_numbers[0..12]: [134, 139, 136, 136, 146, 136, 138, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801069612Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91192024-02-13T21:42:58.039Z{"msg":"[2]R generation[0..12]: [77, 81, 79, 79, 83, 79, 80, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801084616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91202024-02-13T21:42:58.039Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801105239Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91212024-02-13T21:42:58.039Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801118904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91222024-02-13T21:42:58.039Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801138876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91232024-02-13T21:42:58.039Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801169325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91242024-02-13T21:42:58.039Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.80118389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"":"downstairs","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91252024-02-13T21:42:58.040Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801204163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91262024-02-13T21:42:58.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801216618Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"0","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91272024-02-13T21:42:58.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801242049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"1","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91282024-02-13T21:42:58.040Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801255894Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"client":"2","":"downstairs","session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91292024-02-13T21:42:58.040Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801276257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91302024-02-13T21:42:58.040Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801288832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91312024-02-13T21:42:58.040Z{"msg":"b652d5b2-b55e-44f1-b573-ac92f25d8d72 is now active with session: b04a3a8f-a3a4-408b-a278-e18df6c9a54c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801301797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91322024-02-13T21:42:58.040Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801313673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474,"session_id":"b04a3a8f-a3a4-408b-a278-e18df6c9a54c"}
91332024-02-13T21:42:58.040Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:42:56.801325889Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
91342024-02-13T21:42:58.048ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91352024-02-13T21:42:58.048ZLoading write count information from file "/tmp/test_repair_verify.data"
91362024-02-13T21:42:58.048ZRun Repair workload
91372024-02-13T21:42:58.048Z01/30 Write block 50 len 40960 data: 6 6 6 4 5 6 7 8 8 9
91382024-02-13T21:42:58.048Z02/30 Read block 278 len 16384
91392024-02-13T21:42:58.049Z03/30 Write block 454 len 12288 data: 5 6 6
91402024-02-13T21:42:58.049Z04/30 Read block 409 len 24576
91412024-02-13T21:42:58.049Z05/30 Read block 238 len 28672
91422024-02-13T21:42:58.049Z06/30 Read block 430 len 28672
91432024-02-13T21:42:58.050Z07/30 Read block 425 len 40960
91442024-02-13T21:42:58.050Z08/30 Read block 561 len 40960
91452024-02-13T21:42:58.050Z09/30 Write block 303 len 20480 data: 6 6 6 5 8
91462024-02-13T21:42:58.051Z10/30 Read block 481 len 8192
91472024-02-13T21:42:58.051Z11/30 Read block 62 len 8192
91482024-02-13T21:42:58.051Z12/30 Flush
91492024-02-13T21:42:58.051Z13/30 Write block 415 len 12288 data: 9 10 11
91502024-02-13T21:42:58.051Z14/30 Read block 30 len 32768
91512024-02-13T21:42:58.052Z15/30 Read block 182 len 32768
91522024-02-13T21:42:58.052Z16/30 Write block 551 len 16384 data: 9 7 7 7
91532024-02-13T21:42:58.052Z17/30 Write block 318 len 36864 data: 6 8 8 8 8 10 9 11 11
91542024-02-13T21:42:58.056Z18/30 Read block 312 len 40960
91552024-02-13T21:42:58.057Z19/30 Flush
91562024-02-13T21:42:58.057Z20/30 Write block 255 len 40960 data: 8 8 8 7 5 4 4 4 5 7
91572024-02-13T21:42:58.057Z21/30 Write block 371 len 40960 data: 9 9 9 8 10 9 8 8 8 8
91582024-02-13T21:42:58.057Z22/30 Write block 584 len 32768 data: 6 7 7 6 5 6 7 6
91592024-02-13T21:42:58.057Z23/30 Write block 125 len 28672 data: 9 9 8 7 7 4 8
91602024-02-13T21:42:58.057Z24/30 Write block 569 len 16384 data: 9 7 4 5
91612024-02-13T21:42:58.057Z25/30 Write block 242 len 12288 data: 8 7 5
91622024-02-13T21:42:58.057Z26/30 Read block 468 len 12288
91632024-02-13T21:42:58.062Z27/30 Read block 364 len 20480
91642024-02-13T21:42:58.062Z28/30 Read block 298 len 28672
91652024-02-13T21:42:58.062Z29/30 Read block 466 len 32768
91662024-02-13T21:42:58.062Z30/30 Write block 6 len 32768 data: 7 7 6 8 9 11 9 10
91672024-02-13T21:42:58.063Z----------------------------------------------------------------
91682024-02-13T21:42:58.063Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91692024-02-13T21:42:58.063ZStates: Active Active Active
91702024-02-13T21:42:58.063ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91712024-02-13T21:42:58.063Z 1 Acked 1000 Write 10 Done Done Done false
91722024-02-13T21:42:58.063Z 2 Acked 1001 Read 4 Done Done Done false
91732024-02-13T21:42:58.063Z 3 Acked 1002 Write 3 Done Done Done false
91742024-02-13T21:42:58.064Z 4 Acked 1003 Read 6 Done Done Done false
91752024-02-13T21:42:58.064Z 5 Acked 1004 Read 7 Done Done Done false
91762024-02-13T21:42:58.064Z 6 Acked 1005 Read 7 Done Done Done false
91772024-02-13T21:42:58.064Z 7 Acked 1006 Read 10 Done Done Done false
91782024-02-13T21:42:58.064Z 8 Acked 1007 Read 10 Done Sent Done false
91792024-02-13T21:42:58.065Z 9 Acked 1008 Write 5 Done Sent Done false
91802024-02-13T21:42:58.065Z 10 Acked 1009 Read 2 Done Sent Done false
91812024-02-13T21:42:58.065Z 11 Acked 1010 Read 2 Done Sent Done false
91822024-02-13T21:42:58.065Z 12 Acked 1011 Flush 0 Done Sent Done false
91832024-02-13T21:42:58.065Z 13 Acked 1012 Write 3 Done Sent Done false
91842024-02-13T21:42:58.065Z 14 Acked 1013 Read 8 Done Sent Done false
91852024-02-13T21:42:58.065Z 15 Acked 1014 Read 8 Done Sent Done false
91862024-02-13T21:42:58.065Z 16 Acked 1015 Write 4 Done Sent Done false
91872024-02-13T21:42:58.065Z 17 Acked 1016 Write 9 Done Sent Done false
91882024-02-13T21:42:58.066Z 18 Acked 1017 Read 10 Done Sent Done false
91892024-02-13T21:42:58.066Z 19 Acked 1018 Flush 0 Done Sent Done false
91902024-02-13T21:42:58.066Z 20 Acked 1019 Write 10 Done Sent Done false
91912024-02-13T21:42:58.066Z 21 Acked 1020 Write 10 Done Sent Done false
91922024-02-13T21:42:58.066Z 22 Acked 1021 Write 8 Done Sent Done false
91932024-02-13T21:42:58.066Z 23 Acked 1022 Write 7 Done Sent Done false
91942024-02-13T21:42:58.066Z 24 Acked 1023 Write 4 Done Sent Done false
91952024-02-13T21:42:58.067Z 25 Acked 1024 Write 3 Done Sent Done false
91962024-02-13T21:42:58.067Z 26 Acked 1025 Read 3 Done Sent Done false
91972024-02-13T21:42:58.067Z 27 Acked 1026 Read 5 Done Sent Done false
91982024-02-13T21:42:58.067Z 28 Acked 1027 Read 7 Done Sent Done false
91992024-02-13T21:42:58.067Z 29 Acked 1028 Read 8 Done Sent Done false
92002024-02-13T21:42:58.067Z 30 Acked 1029 Write 8 Sent Sent Sent false
92012024-02-13T21:42:58.067Z STATES DS:0 DS:1 DS:2 TOTAL
92022024-02-13T21:42:58.068Z New 0 0 0 0
92032024-02-13T21:42:58.068Z Sent 1 23 1 25
92042024-02-13T21:42:58.068Z Done 29 7 29 65
92052024-02-13T21:42:58.068Z Skipped 0 0 0 0
92062024-02-13T21:42:58.068Z Error 0 0 0 0
92072024-02-13T21:42:58.068ZLast Flush: 1018 0 1018
92082024-02-13T21:42:58.068ZDownstairs last five completed:
92092024-02-13T21:42:58.068ZUpstairs last five completed: 30 29 28 27 26
92102024-02-13T21:42:58.069Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:42:56.832615129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1474}
92112024-02-13T21:42:58.069ZWrote out file "/tmp/test_repair_verify.data"
92122024-02-13T21:42:58.098Z
92132024-02-13T21:42:58.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92142024-02-13T21:42:58.099ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92152024-02-13T21:42:58.099Z 0 000-019 77 77 77 134 134 134 T F T <---
92162024-02-13T21:42:58.099Z 1 020-039 81 81 81 139 139 139 F F F
92172024-02-13T21:42:58.100Z 2 040-059 84 79 84 147 136 147 F T F <---
92182024-02-13T21:42:58.100Z 3 060-079 79 79 79 136 136 136 F F F
92192024-02-13T21:42:58.100Z 4 080-099 83 83 83 146 146 146 F F F
92202024-02-13T21:42:58.100Z 5 100-119 79 79 79 136 136 136 F F F
92212024-02-13T21:42:58.100Z 6 120-139 80 80 80 138 138 138 T F T <---
92222024-02-13T21:42:58.101Z 7 140-159 83 83 83 146 146 146 F F F
92232024-02-13T21:42:58.101Z 8 160-179 62 62 62 107 107 107 F F F
92242024-02-13T21:42:58.101Z 9 180-199 82 82 82 142 142 142 F F F
92252024-02-13T21:42:58.101Z 10 200-219 80 80 80 137 137 137 F F F
92262024-02-13T21:42:58.101Z 11 220-239 82 82 82 143 143 143 F F F
92272024-02-13T21:42:58.101Z 12 240-259 82 82 82 142 142 142 T F T <---
92282024-02-13T21:42:58.102Z 13 260-279 82 82 82 143 143 143 T F T <---
92292024-02-13T21:42:58.102Z 14 280-299 82 82 82 143 143 143 F F F
92302024-02-13T21:42:58.102Z 15 300-319 84 72 84 148 121 148 F F F <---
92312024-02-13T21:42:58.102Z 16 320-339 84 80 84 148 138 148 F F F <---
92322024-02-13T21:42:58.103Z 17 340-359 81 81 81 139 139 139 F F F
92332024-02-13T21:42:58.103Z 18 360-379 83 83 83 146 146 146 T F T <---
92342024-02-13T21:42:58.103Z 19 380-399 79 79 79 136 136 136 T F T <---
92352024-02-13T21:42:58.103Z 20 400-419 84 82 84 148 141 148 F F F <---
92362024-02-13T21:42:58.103Z 21 420-439 80 80 80 138 138 138 F F F
92372024-02-13T21:42:58.103Z 22 440-459 84 82 84 147 144 147 F T F <---
92382024-02-13T21:42:58.103Z 23 460-479 82 82 82 144 144 144 F F F
92392024-02-13T21:42:58.103Z 24 480-499 82 82 82 145 145 145 F F F
92402024-02-13T21:42:58.103Z 25 500-519 78 78 78 135 135 135 F F F
92412024-02-13T21:42:58.103Z 26 520-539 77 77 77 134 134 134 F F F
92422024-02-13T21:42:58.104Z 27 540-559 84 79 84 148 136 148 F F F <---
92432024-02-13T21:42:58.104Z 28 560-579 80 80 80 137 137 137 T F T <---
92442024-02-13T21:42:58.104Z 29 580-599 74 74 74 130 130 130 T F T <---
92452024-02-13T21:42:58.104ZMax gen: 84, Max flush: 148
92462024-02-13T21:42:58.104ZError: Difference in extent metadata found!
92472024-02-13T21:42:58.104ZA Difference in extent metadata is expected here
92482024-02-13T21:42:58.104ZOn loop 42
92492024-02-13T21:42:58.104Z
92502024-02-13T21:42:58.104ZVerifying data now
92512024-02-13T21:43:03.796ZNo change after verify
92522024-02-13T21:43:03.796ZLoop: 42 Downstairs dump after verify (and repair):
92532024-02-13T21:43:03.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92542024-02-13T21:43:03.819Z 0 000-019 85 85 85 149 149 149 F F F
92552024-02-13T21:43:03.819Z 1 020-039 81 81 81 139 139 139 F F F
92562024-02-13T21:43:03.819Z 2 040-059 84 84 84 147 147 147 F F F
92572024-02-13T21:43:03.819Z 3 060-079 79 79 79 136 136 136 F F F
92582024-02-13T21:43:03.820Z 4 080-099 83 83 83 146 146 146 F F F
92592024-02-13T21:43:03.820Z 5 100-119 79 79 79 136 136 136 F F F
92602024-02-13T21:43:03.820Z 6 120-139 85 85 85 149 149 149 F F F
92612024-02-13T21:43:03.820Z 7 140-159 83 83 83 146 146 146 F F F
92622024-02-13T21:43:03.820Z 8 160-179 62 62 62 107 107 107 F F F
92632024-02-13T21:43:03.844Z 9 180-199 82 82 82 142 142 142 F F F
92642024-02-13T21:43:03.845Z 10 200-219 80 80 80 137 137 137 F F F
92652024-02-13T21:43:03.845Z 11 220-239 82 82 82 143 143 143 F F F
92662024-02-13T21:43:03.845Z 12 240-259 85 85 85 149 149 149 F F F
92672024-02-13T21:43:03.845Z 13 260-279 85 85 85 149 149 149 F F F
92682024-02-13T21:43:03.845Z 14 280-299 82 82 82 143 143 143 F F F
92692024-02-13T21:43:03.845Z 15 300-319 84 84 84 148 148 148 F F F
92702024-02-13T21:43:03.845Z 16 320-339 84 84 84 148 148 148 F F F
92712024-02-13T21:43:03.845Z 17 340-359 81 81 81 139 139 139 F F F
92722024-02-13T21:43:03.845Z 18 360-379 85 85 85 149 149 149 F F F
92732024-02-13T21:43:03.845Z 19 380-399 85 85 85 149 149 149 F F F
92742024-02-13T21:43:03.845Z 20 400-419 84 84 84 148 148 148 F F F
92752024-02-13T21:43:03.845Z 21 420-439 80 80 80 138 138 138 F F F
92762024-02-13T21:43:03.846Z 22 440-459 84 84 84 147 147 147 F F F
92772024-02-13T21:43:03.846Z 23 460-479 82 82 82 144 144 144 F F F
92782024-02-13T21:43:03.846Z 24 480-499 82 82 82 145 145 145 F F F
92792024-02-13T21:43:03.846Z 25 500-519 78 78 78 135 135 135 F F F
92802024-02-13T21:43:03.846Z 26 520-539 77 77 77 134 134 134 F F F
92812024-02-13T21:43:03.846Z 27 540-559 84 84 84 148 148 148 F F F
92822024-02-13T21:43:03.846Z 28 560-579 85 85 85 149 149 149 F F F
92832024-02-13T21:43:03.846Z 29 580-599 85 85 85 149 149 149 F F F
92842024-02-13T21:43:03.847ZMax gen: 85, Max flush: 149
92852024-02-13T21:43:03.847Z
92862024-02-13T21:43:03.847ZBegin loop 43
92872024-02-13T21:43:03.847ZDownstairs to restart: 2
92882024-02-13T21:43:04.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.600923145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
92892024-02-13T21:43:04.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601206643Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
92902024-02-13T21:43:04.826ZCrucible runtime is spawned
92912024-02-13T21:43:04.827Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601251686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
92922024-02-13T21:43:04.827Z{"msg":"Crucible c2eff9ff-c123-4bf9-8884-642b2eaa4d6d has session id: 5e40a252-20c5-4921-ac7a-0387746aea9d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601293511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92932024-02-13T21:43:04.827Z{"msg":"Upstairs opts: Upstairs UUID: c2eff9ff-c123-4bf9-8884-642b2eaa4d6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601346592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92942024-02-13T21:43:04.827Z{"msg":"Crucible stats registered with UUID: c2eff9ff-c123-4bf9-8884-642b2eaa4d6d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601388647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92952024-02-13T21:43:04.827Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601438529Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
92962024-02-13T21:43:04.828Z{"msg":"c2eff9ff-c123-4bf9-8884-642b2eaa4d6d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601466448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92972024-02-13T21:43:04.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601507533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92982024-02-13T21:43:04.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601553637Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
92992024-02-13T21:43:04.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60159854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93002024-02-13T21:43:04.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601653831Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93012024-02-13T21:43:04.828Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601693486Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93022024-02-13T21:43:04.828Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601712039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93032024-02-13T21:43:04.829Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60173891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93042024-02-13T21:43:04.829Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601775416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93052024-02-13T21:43:04.829Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601807345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93062024-02-13T21:43:04.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601850249Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93072024-02-13T21:43:04.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601890594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93082024-02-13T21:43:04.830Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.601968026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"io task","client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93092024-02-13T21:43:04.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60261871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93102024-02-13T21:43:04.830Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.602659866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93112024-02-13T21:43:04.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60270159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93122024-02-13T21:43:04.831Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60273137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93132024-02-13T21:43:04.831Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603247302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93142024-02-13T21:43:04.831Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603287878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93152024-02-13T21:43:04.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603574364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93162024-02-13T21:43:04.831Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603644968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93172024-02-13T21:43:04.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603737125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93182024-02-13T21:43:04.832Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603772612Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93192024-02-13T21:43:04.832Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603789746Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93202024-02-13T21:43:04.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603818765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93212024-02-13T21:43:04.832Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60383348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93222024-02-13T21:43:04.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.603847275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93232024-02-13T21:43:04.833Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604141878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93242024-02-13T21:43:04.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604565844Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93252024-02-13T21:43:04.833Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604640617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93262024-02-13T21:43:04.833Z{"msg":"[0]R flush_numbers[0..12]: [149, 139, 147, 136, 146, 136, 149, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604710942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93272024-02-13T21:43:04.833Z{"msg"Wait for a query_work_queue command to finish before sending IO
93282024-02-13T21:43:04.833Z:"[0]R generation[0..12]: [85, 81, 84, 79, 83, 79, 85, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604765272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93292024-02-13T21:43:04.834Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604809246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93302024-02-13T21:43:04.834Z{"msg":"[1]R flush_numbers[0..12]: [149, 139, 147, 136, 146, 136, 149, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604850511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93312024-02-13T21:43:04.834Z{"msg":"[1]R generation[0..12]: [85, 81, 84, 79, 83, 79, 85, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604892686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93322024-02-13T21:43:04.834Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60493473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93332024-02-13T21:43:04.835Z{"msg":"[2]R flush_numbers[0..12]: [149, 139, 147, 136, 146, 136, 149, 146, 107, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.604974346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93342024-02-13T21:43:04.835Z{"msg":"[2]R generation[0..12]: [85, 81, 84, 79, 83, 79, 85, 83, 62, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605020629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93352024-02-13T21:43:04.835Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605059675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93362024-02-13T21:43:04.835Z{"msg":"Max found gen is 86","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93372024-02-13T21:43:04.835Z,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605139966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93382024-02-13T21:43:04.835Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605174733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93392024-02-13T21:43:04.835Z{"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605190568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93402024-02-13T21:43:04.836Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605204063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"":"downstairs","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93412024-02-13T21:43:04.836Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605225345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93422024-02-13T21:43:04.836Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60523808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"0","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93432024-02-13T21:43:04.836Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605251805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"1","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93442024-02-13T21:43:04.836Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.60526575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"client":"2","":"downstairs","session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93452024-02-13T21:43:04.836Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605279335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93462024-02-13T21:43:04.837Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605299088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93472024-02-13T21:43:04.837Z{"msg":"c2eff9ff-c123-4bf9-8884-642b2eaa4d6d is now active with session: 5e40a252-20c5-4921-ac7a-0387746aea9d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605313143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93482024-02-13T21:43:04.838Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605325939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482,"session_id":"5e40a252-20c5-4921-ac7a-0387746aea9d"}
93492024-02-13T21:43:04.838Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:02.605338544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
93502024-02-13T21:43:04.838ZLoading write count information from file "/tmp/test_repair_verify.data"
93512024-02-13T21:43:04.838ZRun Repair workload
93522024-02-13T21:43:04.838Z01/30 Write block 363 len 12288 data: 6 5 4
93532024-02-13T21:43:04.838Z02/30 Write block 160 len 16384 data: 5 6 6 6
93542024-02-13T21:43:04.839Z03/30 Flush
93552024-02-13T21:43:04.839Z04/30 Write block 416 len 32768 data: 11 12 11 11 10 8 7 8
93562024-02-13T21:43:04.839Z05/30 Read block 141 len 40960
93572024-02-13T21:43:04.839Z06/30 Write block 465 len 32768 data: 8 7 8 6 8 8 9 9
93582024-02-13T21:43:04.840Z07/30 Read block 25 len 20480
93592024-02-13T21:43:04.840Z08/30 Write block 4 len 24576 data: 7 8 8 8 7 9
93602024-02-13T21:43:04.840Z09/30 Write block 13 len 12288 data: 11 12 12
93612024-02-13T21:43:04.840Z10/30 Flush
93622024-02-13T21:43:04.840Z11/30 Write block 291 len 36864 data: 8 9 11 11 9 8 8 8 7
93632024-02-13T21:43:04.840Z12/30 Flush
93642024-02-13T21:43:04.841Z13/30 Flush
93652024-02-13T21:43:04.841Z14/30 Write block 582 len 4096 data: 5
93662024-02-13T21:43:04.841Z15/30 Read block 452 len 20480
93672024-02-13T21:43:04.841Z16/30 Read block 307 len 20480
93682024-02-13T21:43:04.841Z17/30 Flush
93692024-02-13T21:43:04.842Z18/30 Flush
93702024-02-13T21:43:04.842Z19/30 Write block 517 len 4096 data: 12
93712024-02-13T21:43:04.842Z20/30 Write block 416 len 16384 data: 12 13 12 12
93722024-02-13T21:43:04.842Z21/30 Read block 458 len 16384
93732024-02-13T21:43:04.842Z22/30 Read block 329 len 8192
93742024-02-13T21:43:04.842Z23/30 Read block 532 len 16384
93752024-02-13T21:43:04.842Z24/30 Write block 76 len 40960 data: 8 8 5 6 6 8 7 7 6 7
93762024-02-13T21:43:04.843Z25/30 Write block 139 len 28672 data: 8 8 9 12 9 9 9
93772024-02-13T21:43:04.843Z26/30 Flush
93782024-02-13T21:43:04.877Z27/30 Write block 94 len 20480 data: 6 6 7 8 9
93792024-02-13T21:43:04.878Z28/30 Read block 406 len 12288
93802024-02-13T21:43:04.878Z29/30 Write block 16 len 8192 data: 15 13
93812024-02-13T21:43:04.878Z30/30 Write block 466 len 12288 data: 8 9 7
93822024-02-13T21:43:04.878Z----------------------------------------------------------------
93832024-02-13T21:43:04.879Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:27
93842024-02-13T21:43:04.879ZStates: Active Active Active
93852024-02-13T21:43:04.879ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93862024-02-13T21:43:04.879Z 4 Acked 1003 Write 8 Done Done Sent false
93872024-02-13T21:43:04.879Z 5 Acked 1004 Read 10 Done Done Sent false
93882024-02-13T21:43:04.880Z 6 Acked 1005 Write 8 Done Done Sent false
93892024-02-13T21:43:04.880Z 7 Acked 1006 Read 5 Done Done Sent false
93902024-02-13T21:43:04.880Z 8 Acked 1007 Write 6 Done Done Sent false
93912024-02-13T21:43:04.880Z 9 Acked 1008 Write 3 Done Done Sent false
93922024-02-13T21:43:04.880Z 10 Acked 1009 Flush 0 Done Done Sent false
93932024-02-13T21:43:04.880Z 11 Acked 1010 Write 9 Done Done Sent false
93942024-02-13T21:43:04.880Z 12 Acked 1011 Flush 0 Done Done Sent false
93952024-02-13T21:43:04.881Z 13 Acked 1012 Flush 0 Done Done Sent false
93962024-02-13T21:43:04.881Z 14 Acked 1013 Write 1 Done Done Sent false
93972024-02-13T21:43:04.881Z 15 Acked 1014 Read 5 Done Done Sent false
93982024-02-13T21:43:04.881Z 16 Acked 1015 Read 5 Done Done Sent false
93992024-02-13T21:43:04.881Z 17 Acked 1016 Flush 0 Done Done Sent false
94002024-02-13T21:43:04.881Z 18 Acked 1017 Flush 0 Done Done Sent false
94012024-02-13T21:43:04.881Z 19 Acked 1018 Write 1 Done Done Sent false
94022024-02-13T21:43:04.882Z 20 Acked 1019 Write 4 Done Done Sent false
94032024-02-13T21:43:04.882Z 21 Acked 1020 Read 4 Done Done Sent false
94042024-02-13T21:43:04.882Z 22 Acked 1021 Read 2 Done Done Sent false
94052024-02-13T21:43:04.882Z 23 Acked 1022 Read 4 Done Done Sent false
94062024-02-13T21:43:04.882Z 24 Acked 1023 Write 10 Done Done Sent false
94072024-02-13T21:43:04.883Z 25 Acked 1024 Write 7 Done Done Sent false
94082024-02-13T21:43:04.883Z 26 Acked 1025 Flush 0 Done Done Sent false
94092024-02-13T21:43:04.883Z 27 Acked 1026 Write 5 Done Done Sent false
94102024-02-13T21:43:04.883Z 28 Acked 1027 Read 3 Done Done Sent false
94112024-02-13T21:43:04.883Z 29 Acked 1028 Write 2 Done Done Sent false
94122024-02-13T21:43:04.884Z 30 Acked 1029 Write 3 Sent Sent Sent false
94132024-02-13T21:43:04.884Z STATES DS:0 DS:1 DS:2 TOTAL
94142024-02-13T21:43:04.884Z New 0 0 0 0
94152024-02-13T21:43:04.884Z Sent 1 1 27 29
94162024-02-13T21:43:04.884Z Done 26 26 0 52
94172024-02-13T21:43:04.884Z Skipped 0 0 0 0
94182024-02-13T21:43:04.885Z Error 0 0 0 0
94192024-02-13T21:43:04.885ZLast Flush: 1025 1025 1002
94202024-02-13T21:43:04.885ZDownstairs last five completed: 1002 1001 1000
94212024-02-13T21:43:04.885ZUpstairs last five completed: 30 29 28 27 26
94222024-02-13T21:43:04.885Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:02.638103109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1482}
94232024-02-13T21:43:04.885ZWrote out file "/tmp/test_repair_verify.data"
94242024-02-13T21:43:04.885Z
94252024-02-13T21:43:04.885ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94262024-02-13T21:43:04.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94272024-02-13T21:43:04.885Z 0 000-019 86 86 85 151 151 149 T T F <---
94282024-02-13T21:43:04.885Z 1 020-039 81 81 81 139 139 139 F F F
94292024-02-13T21:43:04.885Z 2 040-059 84 84 84 147 147 147 F F F
94302024-02-13T21:43:04.886Z 3 060-079 86 86 79 156 156 136 F F F <---
94312024-02-13T21:43:04.886Z 4 080-099 86 86 83 156 156 146 T T F <---
94322024-02-13T21:43:04.886Z 5 100-119 79 79 79 136 136 136 F F F
94332024-02-13T21:43:04.886Z 6 120-139 86 86 85 156 156 149 F F F <---
94342024-02-13T21:43:04.886Z 7 140-159 86 86 83 156 156 146 F F F <---
94352024-02-13T21:43:04.886Z 8 160-179 86 86 86 150 150 150 F F F
94362024-02-13T21:43:04.886Z 9 180-199 82 82 82 142 142 142 F F F
94372024-02-13T21:43:04.886Z 10 200-219 80 80 80 137 137 137 F F F
94382024-02-13T21:43:04.886Z 11 220-239 82 82 82 143 143 143 F F F
94392024-02-13T21:43:04.886Z 12 240-259 85 85 85 149 149 149 F F F
94402024-02-13T21:43:04.886Z 13 260-279 85 85 85 149 149 149 F F F
94412024-02-13T21:43:04.887Z 14 280-299 86 86 82 152 152 143 F F F <---
94422024-02-13T21:43:04.887Z 15 300-319 84 84 84 148 148 148 F F F
94432024-02-13T21:43:04.887Z 16 320-339 84 84 84 148 148 148 F F F
94442024-02-13T21:43:04.887Z 17 340-359 81 81 81 139 139 139 F F F
94452024-02-13T21:43:04.888Z 18 360-379 86 86 86 150 150 150 F F F
94462024-02-13T21:43:04.888Z 19 380-399 85 85 85 149 149 149 F F F
94472024-02-13T21:43:04.888Z 20 400-419 86 86 84 156 156 148 F F F <---
94482024-02-13T21:43:04.888Z 21 420-439 86 86 80 151 151 138 F F F <---
94492024-02-13T21:43:04.888Z 22 440-459 84 84 84 147 147 147 F F F
94502024-02-13T21:43:04.888Z 23 460-479 86 86 82 151 151 144 T T F <---
94512024-02-13T21:43:04.889Z 24 480-499 82 82 82 145 145 145 F F F
94522024-02-13T21:43:04.889Z 25 500-519 86 86 78 156 156 135 F F F <---
94532024-02-13T21:43:04.889Z 26 520-539 77 77 77 134 134 134 F F F
94542024-02-13T21:43:04.889Z 27 540-559 84 84 84 148 148 148 F F F
94552024-02-13T21:43:04.889Z 28 560-579 85 85 85 149 149 149 F F F
94562024-02-13T21:43:04.889Z 29 580-599 86 86 85 154 154 149 F F F <---
94572024-02-13T21:43:04.890ZMax gen: 86, Max flush: 156
94582024-02-13T21:43:04.890ZError: Difference in extent metadata found!
94592024-02-13T21:43:04.890ZA Difference in extent metadata is expected here
94602024-02-13T21:43:04.890ZOn loop 43
94612024-02-13T21:43:04.891Z
94622024-02-13T21:43:04.891ZVerifying data now
94632024-02-13T21:43:08.451ZNo change after verify
94642024-02-13T21:43:08.451ZLoop: 43 Downstairs dump after verify (and repair):
94652024-02-13T21:43:08.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94662024-02-13T21:43:08.465Z 0 000-019 87 87 87 157 157 157 F F F
94672024-02-13T21:43:08.465Z 1 020-039 81 81 81 139 139 139 F F F
94682024-02-13T21:43:08.465Z 2 040-059 84 84 84 147 147 147 F F F
94692024-02-13T21:43:08.465Z 3 060-079 86 86 86 156 156 156 F F F
94702024-02-13T21:43:08.466Z 4 080-099 87 87 87 157 157 157 F F F
94712024-02-13T21:43:08.466Z 5 100-119 79 79 79 136 136 136 F F F
94722024-02-13T21:43:08.466Z 6 120-139 86 86 86 156 156 156 F F F
94732024-02-13T21:43:08.471Z 7 140-159 86 86 86 156 156 156 F F F
94742024-02-13T21:43:08.471Z 8 160-179 86 86 86 150 150 150 F F F
94752024-02-13T21:43:08.471Z 9 180-199 82 82 82 142 142 142 F F F
94762024-02-13T21:43:08.471Z 10 200-219 80 80 80 137 137 137 F F F
94772024-02-13T21:43:08.472Z 11 220-239 82 82 82 143 143 143 F F F
94782024-02-13T21:43:08.472Z 12 240-259 85 85 85 149 149 149 F F F
94792024-02-13T21:43:08.472Z 13 260-279 85 85 85 149 149 149 F F F
94802024-02-13T21:43:08.472Z 14 280-299 86 86 86 152 152 152 F F F
94812024-02-13T21:43:08.472Z 15 300-319 84 84 84 148 148 148 F F F
94822024-02-13T21:43:08.472Z 16 320-339 84 84 84 148 148 148 F F F
94832024-02-13T21:43:08.472Z 17 340-359 81 81 81 139 139 139 F F F
94842024-02-13T21:43:08.473Z 18 360-379 86 86 86 150 150 150 F F F
94852024-02-13T21:43:08.473Z 19 380-399 85 85 85 149 149 149 F F F
94862024-02-13T21:43:08.473Z 20 400-419 86 86 86 156 156 156 F F F
94872024-02-13T21:43:08.473Z 21 420-439 86 86 86 151 151 151 F F F
94882024-02-13T21:43:08.473Z 22 440-459 84 84 84 147 147 147 F F F
94892024-02-13T21:43:08.473Z 23 460-479 87 87 87 157 157 157 F F F
94902024-02-13T21:43:08.474Z 24 480-499 82 82 82 145 145 145 F F F
94912024-02-13T21:43:08.474Z 25 500-519 86 86 86 156 156 156 F F F
94922024-02-13T21:43:08.474Z 26 520-539 77 77 77 134 134 134 F F F
94932024-02-13T21:43:08.474Z 27 540-559 84 84 84 148 148 148 F F F
94942024-02-13T21:43:08.474Z 28 560-579 85 85 85 149 149 149 F F F
94952024-02-13T21:43:08.474Z 29 580-599 86 86 86 154 154 154 F F F
94962024-02-13T21:43:08.475ZMax gen: 87, Max flush: 157
94972024-02-13T21:43:08.475Z
94982024-02-13T21:43:08.475ZBegin loop 44
94992024-02-13T21:43:08.475ZDownstairs to restart: 0
95002024-02-13T21:43:08.486Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.252960754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
95012024-02-13T21:43:08.486Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253205666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
95022024-02-13T21:43:08.494Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253230327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
95032024-02-13T21:43:08.494Z{"msg":"Crucible 9fa70fd4-bd7a-44dc-9537-57ec322bb59d has session id: 83a26058-a422-4890-a92f-6f5c2b2e6cfc","v":0,"name":"crucible","level":30Crucible runtime is spawned
95042024-02-13T21:43:08.494Z,"time":"2024-02-13T21:43:07.25327678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95052024-02-13T21:43:08.494Z{"msg":"Upstairs opts: Upstairs UUID: 9fa70fd4-bd7a-44dc-9537-57ec322bb59d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253347374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95062024-02-13T21:43:08.494Z{"msg":"Crucible stats registered with UUID: 9fa70fd4-bd7a-44dc-9537-57ec322bb59d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.25338554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95072024-02-13T21:43:08.494Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253409801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
95082024-02-13T21:43:08.495Z{"msg":"9fa70fd4-bd7a-44dc-9537-57ec322bb59d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253503947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95092024-02-13T21:43:08.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253541714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95102024-02-13T21:43:08.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253568674Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95112024-02-13T21:43:08.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253602242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95122024-02-13T21:43:08.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253625993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95132024-02-13T21:43:08.495Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253648885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95142024-02-13T21:43:08.495Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253677784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95152024-02-13T21:43:08.496Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253701296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95162024-02-13T21:43:08.496Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253750498Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95172024-02-13T21:43:08.496Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253775649Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95182024-02-13T21:43:08.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253805028Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95192024-02-13T21:43:08.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253831808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95202024-02-13T21:43:08.496Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.253922706Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"io task","client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95212024-02-13T21:43:08.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.254538792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95222024-02-13T21:43:08.496Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.254580237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95232024-02-13T21:43:08.496Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.254596751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95242024-02-13T21:43:08.497Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.254610506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95252024-02-13T21:43:08.497Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255168334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95262024-02-13T21:43:08.497Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255212798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95272024-02-13T21:43:08.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255562561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95282024-02-13T21:43:08.497Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255602116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95292024-02-13T21:43:08.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255786259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95302024-02-13T21:43:08.498Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.25581155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95312024-02-13T21:43:08.498Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255835332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95322024-02-13T21:43:08.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255862522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95332024-02-13T21:43:08.498Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255906176Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95342024-02-13T21:43:08.498Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.255946101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95352024-02-13T21:43:08.498Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256151877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95362024-02-13T21:43:08.498Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256551342Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95372024-02-13T21:43:08.499Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256596925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95382024-02-13T21:43:08.499Z{"msg":"[0]R flush_numbers[0..12]: [157, 139, 147, 156, 157, 136, 156, 156, 150, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256633382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95392024-02-13T21:43:08.499Z{"msg":"[0]R generation[0..12]: [87, 81, 84, 86, 87, 79, 86, 86, 86, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256660062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95402024-02-13T21:43:08.499Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.25669552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95412024-02-13T21:43:08.499Z{"msg":"[1]R flush_numbers[0..12]: [157, 139, 147, 156, 157, 136, 156, 156, 150, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256731477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95422024-02-13T21:43:08.499Z{"msg":"[1]R generation[0..12]: [87, 81, 84, 86, 87, 79, 86, 86, 86, 82, 80, 82]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
95432024-02-13T21:43:08.500Ztime":"2024-02-13T21:43:07.256767014Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95442024-02-13T21:43:08.500Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256808399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95452024-02-13T21:43:08.500Z{"msg":"[2]R flush_numbers[0..12]: [157, 139, 147, 156, 157, 136, 156, 156, 150, 142, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256845115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95462024-02-13T21:43:08.500Z{"msg":"[2]R generation[0..12]: [87, 81, 84, 86, 87, 79, 86, 86, 86, 82, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256883281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95472024-02-13T21:43:08.500Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256920768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95482024-02-13T21:43:08.500Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256947678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95492024-02-13T21:43:08.500Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.256983845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95502024-02-13T21:43:08.501Z{"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257018732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95512024-02-13T21:43:08.501Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257054399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"":"downstairs","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95522024-02-13T21:43:08.501Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.25708014Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95532024-02-13T21:43:08.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"timeRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95542024-02-13T21:43:08.501Z":"2024-02-13T21:43:07.257114068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"0","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95552024-02-13T21:43:08.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257154633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"1","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95562024-02-13T21:43:08.501Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257182223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"client":"2","":"downstairs","session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95572024-02-13T21:43:08.501Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257196508Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95582024-02-13T21:43:08.502Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257215611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95592024-02-13T21:43:08.502Z{"msg":"9fa70fd4-bd7a-44dc-9537-57ec322bb59d is now active with session: 83a26058-a422-4890-a92f-6f5c2b2e6cfc","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257228916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95602024-02-13T21:43:08.502Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.2572474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490,"session_id":"83a26058-a422-4890-a92f-6f5c2b2e6cfc"}
95612024-02-13T21:43:08.502Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:07.257260015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
95622024-02-13T21:43:08.502ZLoading write count information from file "/tmp/test_repair_verify.data"
95632024-02-13T21:43:08.502ZRun Repair workload
95642024-02-13T21:43:08.507Z01/30 Write block 52 len 40960 data: 7 5 6 7 8 9 9 10 8 8
95652024-02-13T21:43:08.508Z02/30 Write block 335 len 36864 data: 12 11 9 9 8 9 8 9 10
95662024-02-13T21:43:08.508Z03/30 Read block 83 len 32768
95672024-02-13T21:43:08.508Z04/30 Read block 53 len 4096
95682024-02-13T21:43:08.508Z05/30 Read block 67 len 24576
95692024-02-13T21:43:08.508Z06/30 Read block 271 len 4096
95702024-02-13T21:43:08.508Z07/30 Flush
95712024-02-13T21:43:08.508Z08/30 Write block 183 len 28672 data: 8 8 11 10 8 10 11
95722024-02-13T21:43:08.508Z09/30 Write block 268 len 12288 data: 7 7 7
95732024-02-13T21:43:08.509Z10/30 Flush
95742024-02-13T21:43:08.509Z11/30 Flush
95752024-02-13T21:43:08.509Z12/30 Flush
95762024-02-13T21:43:08.509Z13/30 Write block 140 len 20480 data: 9 10 13 10 10
95772024-02-13T21:43:08.509Z14/30 Flush
95782024-02-13T21:43:08.509Z15/30 Flush
95792024-02-13T21:43:08.513Z16/30 Write block 558 len 40960 data: 6 4 4 4 5 5 8 9 9 10
95802024-02-13T21:43:08.514Z17/30 Read block 48 len 12288
95812024-02-13T21:43:08.514Z18/30 Read block 509 len 28672
95822024-02-13T21:43:08.514Z19/30 Write block 244 len 4096 data: 6
95832024-02-13T21:43:08.514Z20/30 Write block 445 len 28672 data: 8 8 7 7 8 7 7
95842024-02-13T21:43:08.514Z21/30 Read block 586 len 32768
95852024-02-13T21:43:08.514Z22/30 Write block 442 len 24576 data: 7 6 6 9 9 8
95862024-02-13T21:43:08.518Z23/30 Read block 233 len 4096
95872024-02-13T21:43:08.518Z24/30 Write block 409 len 40960 data: 6 6 9 6 7 8 10 13 14 13
95882024-02-13T21:43:08.519Z25/30 Write block 537 len 28672 data: 5 6 6 8 9 7 7
95892024-02-13T21:43:08.519Z26/30 Read block 388 len 36864
95902024-02-13T21:43:08.519Z27/30 Write block 523 len 4096 data: 12
95912024-02-13T21:43:08.519Z28/30 Read block 308 len 12288
95922024-02-13T21:43:08.524Z29/30 Read block 220 len 12288
95932024-02-13T21:43:08.524Z30/30 Read block 327 len 28672
95942024-02-13T21:43:08.525Z----------------------------------------------------------------
95952024-02-13T21:43:08.525Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95962024-02-13T21:43:08.525ZStates: Active Active Active
95972024-02-13T21:43:08.525ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95982024-02-13T21:43:08.525Z 1 Acked 1000 Write 10 Sent Done Done false
95992024-02-13T21:43:08.525Z 2 Acked 1001 Write 9 Sent Done Done false
96002024-02-13T21:43:08.526Z 3 Acked 1002 Read 8 Sent Done Done false
96012024-02-13T21:43:08.526Z 4 Acked 1003 Read 1 Sent Done Done false
96022024-02-13T21:43:08.526Z 5 Acked 1004 Read 6 Sent Done Done false
96032024-02-13T21:43:08.526Z 6 Acked 1005 Read 1 Sent Done Done false
96042024-02-13T21:43:08.526Z 7 Acked 1006 Flush 0 Sent Done Done false
96052024-02-13T21:43:08.527Z 8 Acked 1007 Write 7 Sent Done Done false
96062024-02-13T21:43:08.527Z 9 Acked 1008 Write 3 Sent Done Done false
96072024-02-13T21:43:08.527Z 10 Acked 1009 Flush 0 Sent Done Done false
96082024-02-13T21:43:08.527Z 11 Acked 1010 Flush 0 Sent Done Done false
96092024-02-13T21:43:08.528Z 12 Acked 1011 Flush 0 Sent Done Done false
96102024-02-13T21:43:08.528Z 13 Acked 1012 Write 5 Sent Done Done false
96112024-02-13T21:43:08.528Z 14 Acked 1013 Flush 0 Sent Done Done false
96122024-02-13T21:43:08.528Z 15 Acked 1014 Flush 0 Sent Done Done false
96132024-02-13T21:43:08.528Z 16 Acked 1015 Write 10 Sent Done Done false
96142024-02-13T21:43:08.528Z 17 Acked 1016 Read 3 Sent Done Done false
96152024-02-13T21:43:08.528Z 18 Acked 1017 Read 7 Sent Done Done false
96162024-02-13T21:43:08.528Z 19 Acked 1018 Write 1 Sent Done Done false
96172024-02-13T21:43:08.529Z 20 Acked 1019 Write 7 Sent Done Done false
96182024-02-13T21:43:08.529Z 21 Acked 1020 Read 8 Sent Done Done false
96192024-02-13T21:43:08.529Z 22 Acked 1021 Write 6 Sent Done Done false
96202024-02-13T21:43:08.529Z 23 Acked 1022 Read 1 Sent Done Done false
96212024-02-13T21:43:08.529Z 24 Acked 1023 Write 10 Sent Done Done false
96222024-02-13T21:43:08.529Z 25 Acked 1024 Write 7 Sent Done Done false
96232024-02-13T21:43:08.530Z 26 Acked 1025 Read 9 Sent Done Done false
96242024-02-13T21:43:08.530Z 27 Acked 1026 Write 1 Sent Done Done false
96252024-02-13T21:43:08.530Z 28 Acked 1027 Read 3 Sent Done Done false
96262024-02-13T21:43:08.530Z 29 Acked 1028 Read 3 Sent Done Done false
96272024-02-13T21:43:08.530Z 30 Acked 1029 Read 7 Sent Done Sent false
96282024-02-13T21:43:08.530Z STATES DS:0 DS:1 DS:2 TOTAL
96292024-02-13T21:43:08.530Z New 0 0 0 0
96302024-02-13T21:43:08.531Z Sent 30 0 1 31
96312024-02-13T21:43:08.531Z Done 0 30 29 59
96322024-02-13T21:43:08.531Z Skipped 0 0 0 0
96332024-02-13T21:43:08.531Z Error 0 0 0 0
96342024-02-13T21:43:08.531ZLast Flush: 0 1014 1014
96352024-02-13T21:43:08.531ZDownstairs last five completed:
96362024-02-13T21:43:08.531ZUpstairs last five completed: 30 29 28 27 26
96372024-02-13T21:43:08.531Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:07.288486668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1490}
96382024-02-13T21:43:08.531ZWrote out file "/tmp/test_repair_verify.data"
96392024-02-13T21:43:08.536Z
96402024-02-13T21:43:08.536ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96412024-02-13T21:43:08.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96422024-02-13T21:43:08.545Z 0 000-019 87 87 87 157 157 157 F F F
96432024-02-13T21:43:08.545Z 1 020-039 81 81 81 139 139 139 F F F
96442024-02-13T21:43:08.546Z 2 040-059 84 88 88 147 158 158 F F F <---
96452024-02-13T21:43:08.546Z 3 060-079 86 88 88 156 158 158 F F F <---
96462024-02-13T21:43:08.546Z 4 080-099 87 87 87 157 157 157 F F F
96472024-02-13T21:43:08.546Z 5 100-119 79 79 79 136 136 136 F F F
96482024-02-13T21:43:08.546Z 6 120-139 86 86 86 156 156 156 F F F
96492024-02-13T21:43:08.546Z 7 140-159 86 88 88 156 162 162 F F F <---
96502024-02-13T21:43:08.547Z 8 160-179 86 86 86 150 150 150 F F F
96512024-02-13T21:43:08.547Z 9 180-199 82 88 88 142 159 159 F F F <---
96522024-02-13T21:43:08.547Z 10 200-219 80 80 80 137 137 137 F F F
96532024-02-13T21:43:08.547Z 11 220-239 82 82 82 143 143 143 F F F
96542024-02-13T21:43:08.552Z 12 240-259 85 85 85 149 149 149 F T T <---
96552024-02-13T21:43:08.552Z 13 260-279 85 88 88 149 159 159 F F F <---
96562024-02-13T21:43:08.552Z 14 280-299 86 86 86 152 152 152 F F F
96572024-02-13T21:43:08.552Z 15 300-319 84 84 84 148 148 148 F F F
96582024-02-13T21:43:08.552Z 16 320-339 84 88 88 148 158 158 F F F <---
96592024-02-13T21:43:08.553Z 17 340-359 81 88 88 139 158 158 F F F <---
96602024-02-13T21:43:08.553Z 18 360-379 86 86 86 150 150 150 F F F
96612024-02-13T21:43:08.553Z 19 380-399 85 85 85 149 149 149 F F F
96622024-02-13T21:43:08.553Z 20 400-419 86 86 86 156 156 156 F T T <---
96632024-02-13T21:43:08.553Z 21 420-439 86 86 86 151 151 151 F F F
96642024-02-13T21:43:08.553Z 22 440-459 84 84 84 147 147 147 F T T <---
96652024-02-13T21:43:08.553Z 23 460-479 87 87 87 157 157 157 F F F
96662024-02-13T21:43:08.554Z 24 480-499 82 82 82 145 145 145 F F F
96672024-02-13T21:43:08.554Z 25 500-519 86 86 86 156 156 156 F F F
96682024-02-13T21:43:08.554Z 26 520-539 77 77 77 134 134 134 F T T <---
96692024-02-13T21:43:08.554Z 27 540-559 84 84 84 148 148 148 F T T <---
96702024-02-13T21:43:08.554Z 28 560-579 85 85 85 149 149 149 F T T <---
96712024-02-13T21:43:08.554Z 29 580-599 86 86 86 154 154 154 F F F
96722024-02-13T21:43:08.554ZMax gen: 88, Max flush: 162
96732024-02-13T21:43:08.554ZError: Difference in extent metadata found!
96742024-02-13T21:43:08.555ZA Difference in extent metadata is expected here
96752024-02-13T21:43:08.555ZOn loop 44
96762024-02-13T21:43:08.555Z
96772024-02-13T21:43:08.555ZVerifying data now
96782024-02-13T21:43:13.169ZNo change after verify
96792024-02-13T21:43:13.169ZLoop: 44 Downstairs dump after verify (and repair):
96802024-02-13T21:43:13.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96812024-02-13T21:43:13.184Z 0 000-019 87 87 87 157 157 157 F F F
96822024-02-13T21:43:13.184Z 1 020-039 81 81 81 139 139 139 F F F
96832024-02-13T21:43:13.184Z 2 040-059 88 88 88 158 158 158 F F F
96842024-02-13T21:43:13.184Z 3 060-079 88 88 88 158 158 158 F F F
96852024-02-13T21:43:13.184Z 4 080-099 87 87 87 157 157 157 F F F
96862024-02-13T21:43:13.184Z 5 100-119 79 79 79 136 136 136 F F F
96872024-02-13T21:43:13.184Z 6 120-139 86 86 86 156 156 156 F F F
96882024-02-13T21:43:13.184Z 7 140-159 88 88 88 162 162 162 F F F
96892024-02-13T21:43:13.185Z 8 160-179 86 86 86 150 150 150 F F F
96902024-02-13T21:43:13.185Z 9 180-199 88 88 88 159 159 159 F F F
96912024-02-13T21:43:13.185Z 10 200-219 80 80 80 137 137 137 F F F
96922024-02-13T21:43:13.185Z 11 220-239 82 82 82 143 143 143 F F F
96932024-02-13T21:43:13.185Z 12 240-259 89 89 89 163 163 163 F F F
96942024-02-13T21:43:13.189Z 13 260-279 88 88 88 159 159 159 F F F
96952024-02-13T21:43:13.189Z 14 280-299 86 86 86 152 152 152 F F F
96962024-02-13T21:43:13.189Z 15 300-319 84 84 84 148 148 148 F F F
96972024-02-13T21:43:13.189Z 16 320-339 88 88 88 158 158 158 F F F
96982024-02-13T21:43:13.189Z 17 340-359 88 88 88 158 158 158 F F F
96992024-02-13T21:43:13.189Z 18 360-379 86 86 86 150 150 150 F F F
97002024-02-13T21:43:13.189Z 19 380-399 85 85 85 149 149 149 F F F
97012024-02-13T21:43:13.189Z 20 400-419 89 89 89 163 163 163 F F F
97022024-02-13T21:43:13.189Z 21 420-439 86 86 86 151 151 151 F F F
97032024-02-13T21:43:13.190Z 22 440-459 89 89 89 163 163 163 F F F
97042024-02-13T21:43:13.190Z 23 460-479 87 87 87 157 157 157 F F F
97052024-02-13T21:43:13.190Z 24 480-499 82 82 82 145 145 145 F F F
97062024-02-13T21:43:13.190Z 25 500-519 86 86 86 156 156 156 F F F
97072024-02-13T21:43:13.190Z 26 520-539 89 89 89 163 163 163 F F F
97082024-02-13T21:43:13.190Z 27 540-559 89 89 89 163 163 163 F F F
97092024-02-13T21:43:13.190Z 28 560-579 89 89 89 163 163 163 F F F
97102024-02-13T21:43:13.190Z 29 580-599 86 86 86 154 154 154 F F F
97112024-02-13T21:43:13.190ZMax gen: 89, Max flush: 163
97122024-02-13T21:43:13.190Z
97132024-02-13T21:43:13.190ZBegin loop 45
97142024-02-13T21:43:13.190ZDownstairs to restart: 2
97152024-02-13T21:43:13.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97086381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
97162024-02-13T21:43:13.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971129814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
97172024-02-13T21:43:13.212ZCrucible runtime is spawned
97182024-02-13T21:43:13.212Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971263675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
97192024-02-13T21:43:13.212Z{"msg":"Crucible 6b054264-867f-4a7e-acc9-5cec6ea2c867 has session id: c426343d-c4f6-43a7-8f18-c2d336134b37","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971309269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97202024-02-13T21:43:13.212Z{"msg":"Upstairs opts: Upstairs UUID: 6b054264-867f-4a7e-acc9-5cec6ea2c867, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97136024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97212024-02-13T21:43:13.212Z{"msg":"Crucible stats registered with UUID: 6b054264-867f-4a7e-acc9-5cec6ea2c867","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971381493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97222024-02-13T21:43:13.212Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971406823Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
97232024-02-13T21:43:13.212Z{"msg":"6b054264-867f-4a7e-acc9-5cec6ea2c867 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971421648Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97242024-02-13T21:43:13.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971450188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97252024-02-13T21:43:13.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971482496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97262024-02-13T21:43:13.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971514544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97272024-02-13T21:43:13.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971548282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97282024-02-13T21:43:13.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97158202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97292024-02-13T21:43:13.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971613269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97302024-02-13T21:43:13.213Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97163521Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97312024-02-13T21:43:13.213Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971707814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97322024-02-13T21:43:13.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971796382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97332024-02-13T21:43:13.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971831429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97342024-02-13T21:43:13.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971855181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97352024-02-13T21:43:13.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.971875933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"io task","client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97362024-02-13T21:43:13.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.972327039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97372024-02-13T21:43:13.213Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.972366885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97382024-02-13T21:43:13.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.972392366Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97392024-02-13T21:43:13.214Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.972426394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97402024-02-13T21:43:13.214Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.972954662Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97412024-02-13T21:43:13.214Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973023697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97422024-02-13T21:43:13.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973214058Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97432024-02-13T21:43:13.214Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97326188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97442024-02-13T21:43:13.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973588112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97452024-02-13T21:43:13.214Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973625868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97462024-02-13T21:43:13.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973642772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97472024-02-13T21:43:13.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973673841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97482024-02-13T21:43:13.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973708609Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97492024-02-13T21:43:13.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973741037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97502024-02-13T21:43:13.214Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.973838891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97512024-02-13T21:43:13.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974247743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97522024-02-13T21:43:13.214Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974288718Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97532024-02-13T21:43:13.215Z{"msg":"[0]R flush_numbers[0..12]: [157, 139, 158, 158, 157, 136, 156, 162, 150, 159, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974306472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97542024-02-13T21:43:13.215Z{"msg":"[0]R generation[0..12]: [87, 81, 88, 88, 87, 79, 86, 88, 86, 88, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974328414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97552024-02-13T21:43:13.215Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974343008Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97562024-02-13T21:43:13.215Z{"msg":"[1]R flush_numbers[0..12]: [157, 139, 158, 158, 157, 136, 156, 162, 150, 159, 137, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.97436612Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97572024-02-13T21:43:13.215Z{"msg":"[1]R generation[0..12]: [87, 81, 88, 88, 87, 79, 86, 88, 86, 88, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974381074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97582024-02-13T21:43:13.215Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974402197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97592024-02-13T21:43:13.215Z{"msg":"[2]R flush_numbers[0..12]: [157, 139, 158, 158, 157, 136, 156, 162, 150, 159, 137, 143]","vWait for a query_work_queue command to finish before sending IO
97602024-02-13T21:43:13.215Z":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974418811Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97612024-02-13T21:43:13.215Z{"msg":"[2]R generation[0..12]: [87, 81, 88, 88, 87, 79, 86, 88, 86, 88, 80, 82]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974439313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97622024-02-13T21:43:13.215Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974453878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97632024-02-13T21:43:13.215Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974473481Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97642024-02-13T21:43:13.215Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974487046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97652024-02-13T21:43:13.215Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974500891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97662024-02-13T21:43:13.216Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974514126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"":"downstairs","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97672024-02-13T21:43:13.216Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974528011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97682024-02-13T21:43:13.216Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974547304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"0","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97692024-02-13T21:43:13.216Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974560839Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"1","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97702024-02-13T21:43:13.216Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974574854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"client":"2","":"downstairs","session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97712024-02-13T21:43:13.216Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974594737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97722024-02-13T21:43:13.216Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974607283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97732024-02-13T21:43:13.216Z{"msg":"6b054264-867f-4a7e-acc9-5cec6ea2c867 is now active with session: c426343d-c4f6-43a7-8f18-c2d336134b37","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974620648Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97742024-02-13T21:43:13.216Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974638811Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498,"session_id":"c426343d-c4f6-43a7-8f18-c2d336134b37"}
97752024-02-13T21:43:13.216Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:11.974651247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
97762024-02-13T21:43:13.216ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97772024-02-13T21:43:13.220ZLoading write count information from file "/tmp/test_repair_verify.data"
97782024-02-13T21:43:13.221ZRun Repair workload
97792024-02-13T21:43:13.221Z01/30 Write block 449 len 8192 data: 9 8
97802024-02-13T21:43:13.221Z02/30 Flush
97812024-02-13T21:43:13.221Z03/30 Write block 175 len 40960 data: 5 4 7 6 5 6 5 5 9 9
97822024-02-13T21:43:13.221Z04/30 Read block 337 len 8192
97832024-02-13T21:43:13.221Z05/30 Read block 437 len 20480
97842024-02-13T21:43:13.221Z06/30 Flush
97852024-02-13T21:43:13.221Z07/30 Write block 378 len 36864 data: 9 9 9 10 10 8 6 6 7
97862024-02-13T21:43:13.221Z08/30 Write block 464 len 8192 data: 9 9
97872024-02-13T21:43:13.221Z09/30 Read block 283 len 32768
97882024-02-13T21:43:13.221Z10/30 Write block 215 len 36864 data: 7 9 10 11 10 8 8 9 9
97892024-02-13T21:43:13.225Z11/30 Read block 483 len 36864
97902024-02-13T21:43:13.225Z12/30 Read block 91 len 12288
97912024-02-13T21:43:13.225Z13/30 Read block 43 len 28672
97922024-02-13T21:43:13.225Z14/30 Write block 23 len 8192 data: 7 7
97932024-02-13T21:43:13.225Z15/30 Read block 81 len 36864
97942024-02-13T21:43:13.234Z16/30 Read block 399 len 20480
97952024-02-13T21:43:13.234Z17/30 Flush
97962024-02-13T21:43:13.242Z18/30 Write block 539 len 36864 data: 7 9 10 8 8 8 10 9 8
97972024-02-13T21:43:13.242Z19/30 Read block 487 len 28672
97982024-02-13T21:43:13.243Z20/30 Flush
97992024-02-13T21:43:13.243Z21/30 Write block 125 len 12288 data: 10 10 9
98002024-02-13T21:43:13.243Z22/30 Read block 75 len 8192
98012024-02-13T21:43:13.243Z23/30 Read block 527 len 40960
98022024-02-13T21:43:13.243Z24/30 Read block 106 len 16384
98032024-02-13T21:43:13.243Z25/30 Flush
98042024-02-13T21:43:13.253Z26/30 Write block 529 len 20480 data: 5 5 4 5 6
98052024-02-13T21:43:13.253Z27/30 Read block 540 len 16384
98062024-02-13T21:43:13.253Z28/30 Write block 560 len 8192 data: 5 5
98072024-02-13T21:43:13.253Z29/30 Write block 279 len 4096 data: 11
98082024-02-13T21:43:13.253Z30/30 Read block 275 len 24576
98092024-02-13T21:43:13.253Z----------------------------------------------------------------
98102024-02-13T21:43:13.253Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:24
98112024-02-13T21:43:13.253ZStates: Active Active Active
98122024-02-13T21:43:13.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98132024-02-13T21:43:13.253Z 7 Acked 1006 Write 9 Done Done Sent false
98142024-02-13T21:43:13.253Z 8 Acked 1007 Write 2 Done Done Sent false
98152024-02-13T21:43:13.253Z 9 Acked 1008 Read 8 Done Done Sent false
98162024-02-13T21:43:13.253Z 10 Acked 1009 Write 9 Done Done Sent false
98172024-02-13T21:43:13.253Z 11 Acked 1010 Read 9 Done Done Sent false
98182024-02-13T21:43:13.253Z 12 Acked 1011 Read 3 Done Done Sent false
98192024-02-13T21:43:13.253Z 13 Acked 1012 Read 7 Done Done Sent false
98202024-02-13T21:43:13.253Z 14 Acked 1013 Write 2 Done Done Sent false
98212024-02-13T21:43:13.253Z 15 Acked 1014 Read 9 Done Done Sent false
98222024-02-13T21:43:13.253Z 16 Acked 1015 Read 5 Done Done Sent false
98232024-02-13T21:43:13.253Z 17 Acked 1016 Flush 0 Done Done Sent false
98242024-02-13T21:43:13.253Z 18 Acked 1017 Write 9 Done Done Sent false
98252024-02-13T21:43:13.253Z 19 Acked 1018 Read 7 Done Done Sent false
98262024-02-13T21:43:13.253Z 20 Acked 1019 Flush 0 Done Done Sent false
98272024-02-13T21:43:13.253Z 21 Acked 1020 Write 3 Done Done Sent false
98282024-02-13T21:43:13.254Z 22 Acked 1021 Read 2 Done Done Sent false
98292024-02-13T21:43:13.254Z 23 Acked 1022 Read 10 Done Done Sent false
98302024-02-13T21:43:13.254Z 24 Acked 1023 Read 4 Done Done Sent false
98312024-02-13T21:43:13.254Z 25 Acked 1024 Flush 0 Done Done Sent false
98322024-02-13T21:43:13.254Z 26 Acked 1025 Write 5 Done Done Sent false
98332024-02-13T21:43:13.254Z 27 Acked 1026 Read 4 Done Done Sent false
98342024-02-13T21:43:13.254Z 28 Acked 1027 Write 2 Done Done Sent false
98352024-02-13T21:43:13.254Z 29 Acked 1028 Write 1 Done Done Sent false
98362024-02-13T21:43:13.254Z 30 Acked 1029 Read 6 Done Sent Sent false
98372024-02-13T21:43:13.254Z STATES DS:0 DS:1 DS:2 TOTAL
98382024-02-13T21:43:13.254Z New 0 0 0 0
98392024-02-13T21:43:13.254Z Sent 0 1 24 25
98402024-02-13T21:43:13.254Z Done 24 23 0 47
98412024-02-13T21:43:13.254Z Skipped 0 0 0 0
98422024-02-13T21:43:13.254Z Error 0 0 0 0
98432024-02-13T21:43:13.254ZLast Flush: 1024 1024 1005
98442024-02-13T21:43:13.254ZDownstairs last five completed: 1005 1004 1003 1002 1001
98452024-02-13T21:43:13.255ZUpstairs last five completed: 30 29 28 27 26
98462024-02-13T21:43:13.255Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:12.008078501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1498}
98472024-02-13T21:43:13.255ZWrote out file "/tmp/test_repair_verify.data"
98482024-02-13T21:43:13.255Z
98492024-02-13T21:43:13.265ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98502024-02-13T21:43:13.273ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98512024-02-13T21:43:13.273Z 0 000-019 87 87 87 157 157 157 F F F
98522024-02-13T21:43:13.273Z 1 020-039 90 90 81 166 166 139 F F F <---
98532024-02-13T21:43:13.273Z 2 040-059 88 88 88 158 158 158 F F F
98542024-02-13T21:43:13.273Z 3 060-079 88 88 88 158 158 158 F F F
98552024-02-13T21:43:13.273Z 4 080-099 87 87 87 157 157 157 F F F
98562024-02-13T21:43:13.274Z 5 100-119 79 79 79 136 136 136 F F F
98572024-02-13T21:43:13.274Z 6 120-139 90 90 86 168 168 156 F F F <---
98582024-02-13T21:43:13.274Z 7 140-159 88 88 88 162 162 162 F F F
98592024-02-13T21:43:13.274Z 8 160-179 90 90 90 165 165 165 F F F
98602024-02-13T21:43:13.274Z 9 180-199 90 90 90 165 165 165 F F F
98612024-02-13T21:43:13.274Z 10 200-219 90 90 80 166 166 137 F F F <---
98622024-02-13T21:43:13.274Z 11 220-239 90 90 82 166 166 143 F F F <---
98632024-02-13T21:43:13.274Z 12 240-259 89 89 89 163 163 163 F F F
98642024-02-13T21:43:13.274Z 13 260-279 88 88 88 159 159 159 T T F <---
98652024-02-13T21:43:13.274Z 14 280-299 86 86 86 152 152 152 F F F
98662024-02-13T21:43:13.274Z 15 300-319 84 84 84 148 148 148 F F F
98672024-02-13T21:43:13.274Z 16 320-339 88 88 88 158 158 158 F F F
98682024-02-13T21:43:13.274Z 17 340-359 88 88 88 158 158 158 F F F
98692024-02-13T21:43:13.275Z 18 360-379 90 90 86 166 166 150 F F F <---
98702024-02-13T21:43:13.275Z 19 380-399 90 90 85 166 166 149 F F F <---
98712024-02-13T21:43:13.275Z 20 400-419 89 89 89 163 163 163 F F F
98722024-02-13T21:43:13.275Z 21 420-439 86 86 86 151 151 151 F F F
98732024-02-13T21:43:13.275Z 22 440-459 90 90 90 164 164 164 F F F
98742024-02-13T21:43:13.275Z 23 460-479 90 90 87 166 166 157 F F F <---
98752024-02-13T21:43:13.275Z 24 480-499 82 82 82 145 145 145 F F F
98762024-02-13T21:43:13.275Z 25 500-519 86 86 86 156 156 156 F F F
98772024-02-13T21:43:13.275Z 26 520-539 90 90 89 167 167 163 T T F <---
98782024-02-13T21:43:13.275Z 27 540-559 90 90 89 167 167 163 F F F <---
98792024-02-13T21:43:13.275Z 28 560-579 89 89 89 163 163 163 T T F <---
98802024-02-13T21:43:13.275Z 29 580-599 86 86 86 154 154 154 F F F
98812024-02-13T21:43:13.276ZMax gen: 90, Max flush: 168
98822024-02-13T21:43:13.276ZError: Difference in extent metadata found!
98832024-02-13T21:43:13.276ZA Difference in extent metadata is expected here
98842024-02-13T21:43:13.276ZOn loop 45
98852024-02-13T21:43:13.276Z
98862024-02-13T21:43:13.282ZVerifying data now
98872024-02-13T21:43:17.825ZNo change after verify
98882024-02-13T21:43:17.825ZLoop: 45 Downstairs dump after verify (and repair):
98892024-02-13T21:43:17.839ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98902024-02-13T21:43:17.840Z 0 000-019 87 87 87 157 157 157 F F F
98912024-02-13T21:43:17.840Z 1 020-039 90 90 90 166 166 166 F F F
98922024-02-13T21:43:17.840Z 2 040-059 88 88 88 158 158 158 F F F
98932024-02-13T21:43:17.840Z 3 060-079 88 88 88 158 158 158 F F F
98942024-02-13T21:43:17.840Z 4 080-099 87 87 87 157 157 157 F F F
98952024-02-13T21:43:17.845Z 5 100-119 79 79 79 136 136 136 F F F
98962024-02-13T21:43:17.845Z 6 120-139 90 90 90 168 168 168 F F F
98972024-02-13T21:43:17.845Z 7 140-159 88 88 88 162 162 162 F F F
98982024-02-13T21:43:17.845Z 8 160-179 90 90 90 165 165 165 F F F
98992024-02-13T21:43:17.846Z 9 180-199 90 90 90 165 165 165 F F F
99002024-02-13T21:43:17.846Z 10 200-219 90 90 90 166 166 166 F F F
99012024-02-13T21:43:17.846Z 11 220-239 90 90 90 166 166 166 F F F
99022024-02-13T21:43:17.846Z 12 240-259 89 89 89 163 163 163 F F F
99032024-02-13T21:43:17.846Z 13 260-279 91 91 91 169 169 169 F F F
99042024-02-13T21:43:17.846Z 14 280-299 86 86 86 152 152 152 F F F
99052024-02-13T21:43:17.847Z 15 300-319 84 84 84 148 148 148 F F F
99062024-02-13T21:43:17.847Z 16 320-339 88 88 88 158 158 158 F F F
99072024-02-13T21:43:17.847Z 17 340-359 88 88 88 158 158 158 F F F
99082024-02-13T21:43:17.847Z 18 360-379 90 90 90 166 166 166 F F F
99092024-02-13T21:43:17.847Z 19 380-399 90 90 90 166 166 166 F F F
99102024-02-13T21:43:17.847Z 20 400-419 89 89 89 163 163 163 F F F
99112024-02-13T21:43:17.847Z 21 420-439 86 86 86 151 151 151 F F F
99122024-02-13T21:43:17.848Z 22 440-459 90 90 90 164 164 164 F F F
99132024-02-13T21:43:17.848Z 23 460-479 90 90 90 166 166 166 F F F
99142024-02-13T21:43:17.848Z 24 480-499 82 82 82 145 145 145 F F F
99152024-02-13T21:43:17.848Z 25 500-519 86 86 86 156 156 156 F F F
99162024-02-13T21:43:17.849Z 26 520-539 91 91 91 169 169 169 F F F
99172024-02-13T21:43:17.849Z 27 540-559 90 90 90 167 167 167 F F F
99182024-02-13T21:43:17.849Z 28 560-579 91 91 91 169 169 169 F F F
99192024-02-13T21:43:17.849Z 29 580-599 86 86 86 154 154 154 F F F
99202024-02-13T21:43:17.849ZMax gen: 91, Max flush: 169
99212024-02-13T21:43:17.849Z
99222024-02-13T21:43:17.850ZBegin loop 46
99232024-02-13T21:43:17.850ZDownstairs to restart: 0
99242024-02-13T21:43:17.860Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.623646167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
99252024-02-13T21:43:17.861Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62388767Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
99262024-02-13T21:43:17.888Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.623912181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
99272024-02-13T21:43:17.889Z{"msg":"Crucible ad6d1d32-b519-4fd7-83c7-45b6ece57fca has session id: 698653b0-4901-4b3f-86a5-cc36e71fa411","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.623954316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99282024-02-13T21:43:17.889Z{"msg":"Crucible runtime is spawned
99292024-02-13T21:43:17.889ZUpstairs opts: Upstairs UUID: ad6d1d32-b519-4fd7-83c7-45b6ece57fca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.623991772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99302024-02-13T21:43:17.889Z{"msg":"Crucible stats registered with UUID: ad6d1d32-b519-4fd7-83c7-45b6ece57fca","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624046452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99312024-02-13T21:43:17.890Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624071133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
99322024-02-13T21:43:17.890Z{"msg":"ad6d1d32-b519-4fd7-83c7-45b6ece57fca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624215231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99332024-02-13T21:43:17.890Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624257356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99342024-02-13T21:43:17.890Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624293433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99352024-02-13T21:43:17.891Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624320543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99362024-02-13T21:43:17.891Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624360378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99372024-02-13T21:43:17.891Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624386389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99382024-02-13T21:43:17.891Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62441119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99392024-02-13T21:43:17.891Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62443809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99402024-02-13T21:43:17.892Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624481514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99412024-02-13T21:43:17.892Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624519421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99422024-02-13T21:43:17.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624546891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99432024-02-13T21:43:17.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624581068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99442024-02-13T21:43:17.892Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.624711481Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"io task","client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99452024-02-13T21:43:17.892Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.625333156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99462024-02-13T21:43:17.892Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.625370512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99472024-02-13T21:43:17.893Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.625387326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99482024-02-13T21:43:17.893Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.625468716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99492024-02-13T21:43:17.893Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.625910986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99502024-02-13T21:43:17.893Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626048666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99512024-02-13T21:43:17.893Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626185156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99522024-02-13T21:43:17.893Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626217025Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99532024-02-13T21:43:17.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626473912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99542024-02-13T21:43:17.894Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626503221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99552024-02-13T21:43:17.894Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626518355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99562024-02-13T21:43:17.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626540957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99572024-02-13T21:43:17.895Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626555282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99582024-02-13T21:43:17.895Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626586031Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99592024-02-13T21:43:17.895Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.626834851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99602024-02-13T21:43:17.895Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62730459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99612024-02-13T21:43:17.895Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627350714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99622024-02-13T21:43:17.895Z{"msg":"[0]R flush_numbers[0..12]: [157, 166, 158, 158, 157, 136, 168, 162, 165, 165, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627376634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99632024-02-13T21:43:17.896Z{"msg":"[0]R generation[0..12]: [87, 90, 88, 88, 87, 79, 90, 88, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627400396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99642024-02-13T21:43:17.896Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627436573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99652024-02-13T21:43:17.896Z{"msg":"[1]R flush_numbers[0..12]: [157, 166, 158, 158, 157, 136, 168, 162, 165, 165, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62747196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99662024-02-13T21:43:17.897Z{"msg":"[1]R generation[0..12]: [87, 90, 88, 88, 87, 79, 90, 88, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62749846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99672024-02-13T21:43:17.897ZWait for a query_work_queue command to finish before sending IO
99682024-02-13T21:43:17.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627540455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99692024-02-13T21:43:17.898Z{"msg":"[2]R flush_numbers[0..12]: [157, 166, 158, 158, 157, 136, 168, 162, 165, 165, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62758108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99702024-02-13T21:43:17.898Z{"msg":"[2]R generation[0..12]: [87, 90, 88, 88, 87, 79, 90, 88, 90, 90, 90, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627619036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99712024-02-13T21:43:17.899Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627659022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99722024-02-13T21:43:17.899Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627694829Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99732024-02-13T21:43:17.899Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627730766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99742024-02-13T21:43:17.899Z{"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627765703Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99752024-02-13T21:43:17.899Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62780119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"":"downstairs","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99762024-02-13T21:43:17.899ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99772024-02-13T21:43:17.900Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627839036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99782024-02-13T21:43:17.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627874863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"0","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99792024-02-13T21:43:17.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627900634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"1","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99802024-02-13T21:43:17.900Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627914609Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"client":"2","":"downstairs","session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99812024-02-13T21:43:17.901Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627934462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99822024-02-13T21:43:17.901Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627947487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99832024-02-13T21:43:17.901Z{"msg":"ad6d1d32-b519-4fd7-83c7-45b6ece57fca is now active with session: 698653b0-4901-4b3f-86a5-cc36e71fa411","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.62796738Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99842024-02-13T21:43:17.901Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627979596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506,"session_id":"698653b0-4901-4b3f-86a5-cc36e71fa411"}
99852024-02-13T21:43:17.901Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:16.627998049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
99862024-02-13T21:43:17.901ZLoading write count information from file "/tmp/test_repair_verify.data"
99872024-02-13T21:43:17.901ZRun Repair workload
99882024-02-13T21:43:17.901Z01/30 Write block 29 len 32768 data: 6 5 8 9 8 9 11 11
99892024-02-13T21:43:17.902Z02/30 Read block 185 len 4096
99902024-02-13T21:43:17.912Z03/30 Write block 74 len 8192 data: 8 8
99912024-02-13T21:43:17.912Z04/30 Read block 49 len 40960
99922024-02-13T21:43:17.913Z05/30 Write block 394 len 24576 data: 6 4 4 3 2 2
99932024-02-13T21:43:17.913Z06/30 Read block 422 len 20480
99942024-02-13T21:43:17.913Z07/30 Write block 177 len 20480 data: 8 7 6 7 6
99952024-02-13T21:43:17.913Z08/30 Read block 56 len 8192
99962024-02-13T21:43:17.913Z09/30 Read block 559 len 24576
99972024-02-13T21:43:17.913Z10/30 Write block 422 len 16384 data: 8 9 7 7
99982024-02-13T21:43:17.914Z11/30 Write block 111 len 28672 data: 6 6 6 5 5 7 8
99992024-02-13T21:43:17.914Z12/30 Read block 211 len 4096
100002024-02-13T21:43:17.914Z13/30 Read block 382 len 40960
100012024-02-13T21:43:17.914Z14/30 Read block 556 len 16384
100022024-02-13T21:43:17.914Z15/30 Write block 311 len 36864 data: 9 11 9 8 7 8 7 7 9
100032024-02-13T21:43:17.914Z16/30 Write block 315 len 40960 data: 8 9 8 8 10 9 9 9 11 10
100042024-02-13T21:43:17.914Z17/30 Write block 402 len 16384 data: 4 5 5 5
100052024-02-13T21:43:17.915Z18/30 Write block 100 len 40960 data: 8 10 14 12 10 9 7 7 7 6
100062024-02-13T21:43:17.915Z19/30 Read block 320 len 24576
100072024-02-13T21:43:17.915Z20/30 Read block 146 len 4096
100082024-02-13T21:43:17.915Z21/30 Read block 137 len 20480
100092024-02-13T21:43:17.915Z22/30 Read block 392 len 4096
100102024-02-13T21:43:17.915Z23/30 Read block 508 len 40960
100112024-02-13T21:43:17.915Z24/30 Write block 550 len 28672 data: 8 10 8 8 8 7 8
100122024-02-13T21:43:17.915Z25/30 Read block 522 len 36864
100132024-02-13T21:43:17.916Z26/30 Write block 238 len 16384 data: 10 10 10 9
100142024-02-13T21:43:17.916Z27/30 Write block 596 len 16384 data: 6 5 3 2
100152024-02-13T21:43:17.916Z28/30 Write block 361 len 20480 data: 6 6 7 6 5
100162024-02-13T21:43:17.916Z29/30 Write block 135 len 16384 data: 7 6 6 9
100172024-02-13T21:43:17.916Z30/30 Read block 339 len 8192
100182024-02-13T21:43:17.916Z----------------------------------------------------------------
100192024-02-13T21:43:17.916Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
100202024-02-13T21:43:17.916ZStates: Active Active Active
100212024-02-13T21:43:17.917ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100222024-02-13T21:43:17.917Z 1 Acked 1000 Write 8 Sent Done Done false
100232024-02-13T21:43:17.917Z 2 Acked 1001 Read 1 Sent Done Done false
100242024-02-13T21:43:17.917Z 3 Acked 1002 Write 2 Sent Done Done false
100252024-02-13T21:43:17.917Z 4 Acked 1003 Read 10 Sent Done Done false
100262024-02-13T21:43:17.917Z 5 Acked 1004 Write 6 Sent Done Done false
100272024-02-13T21:43:17.917Z 6 Acked 1005 Read 5 Sent Done Done false
100282024-02-13T21:43:17.917Z 7 Acked 1006 Write 5 Sent Done Done false
100292024-02-13T21:43:17.917Z 8 Acked 1007 Read 2 Sent Done Done false
100302024-02-13T21:43:17.918Z 9 Acked 1008 Read 6 Sent Done Done false
100312024-02-13T21:43:17.918Z 10 Acked 1009 Write 4 Sent Done Done false
100322024-02-13T21:43:17.918Z 11 Acked 1010 Write 7 Sent Done Done false
100332024-02-13T21:43:17.918Z 12 Acked 1011 Read 1 Sent Done Done false
100342024-02-13T21:43:17.918Z 13 Acked 1012 Read 10 Sent Done Done false
100352024-02-13T21:43:17.918Z 14 Acked 1013 Read 4 Sent Done Done false
100362024-02-13T21:43:17.919Z 15 Acked 1014 Write 9 Sent Done Done false
100372024-02-13T21:43:17.919Z 16 Acked 1015 Write 10 Sent Done Done false
100382024-02-13T21:43:17.919Z 17 Acked 1016 Write 4 Sent Done Done false
100392024-02-13T21:43:17.919Z 18 Acked 1017 Write 10 Sent Done Done false
100402024-02-13T21:43:17.919Z 19 Acked 1018 Read 6 Sent Done Done false
100412024-02-13T21:43:17.919Z 20 Acked 1019 Read 1 Sent Done Done false
100422024-02-13T21:43:17.920Z 21 Acked 1020 Read 5 Sent Done Done false
100432024-02-13T21:43:17.920Z 22 Acked 1021 Read 1 Sent Done Done false
100442024-02-13T21:43:17.920Z 23 Acked 1022 Read 10 Sent Done Done false
100452024-02-13T21:43:17.920Z 24 Acked 1023 Write 7 Sent Done Done false
100462024-02-13T21:43:17.920Z 25 Acked 1024 Read 9 Sent Done Done false
100472024-02-13T21:43:17.920Z 26 Acked 1025 Write 4 Sent Done Done false
100482024-02-13T21:43:17.920Z 27 Acked 1026 Write 4 Sent Done Done false
100492024-02-13T21:43:17.920Z 28 Acked 1027 Write 5 Sent Done Done false
100502024-02-13T21:43:17.921Z 29 Acked 1028 Write 4 Sent Done Done false
100512024-02-13T21:43:17.921Z 30 Acked 1029 Read 2 Sent Sent Done false
100522024-02-13T21:43:17.921Z STATES DS:0 DS:1 DS:2 TOTAL
100532024-02-13T21:43:17.921Z New 0 0 0 0
100542024-02-13T21:43:17.921Z Sent 30 1 0 31
100552024-02-13T21:43:17.921Z Done 0 29 30 59
100562024-02-13T21:43:17.921Z Skipped 0 0 0 0
100572024-02-13T21:43:17.921Z Error 0 0 0 0
100582024-02-13T21:43:17.922ZLast Flush: 0 0 0
100592024-02-13T21:43:17.922ZDownstairs last five completed:
100602024-02-13T21:43:17.922ZUpstairs last five completed: 30 29 28 27 26
100612024-02-13T21:43:17.922Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:16.655972062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1506}
100622024-02-13T21:43:17.922ZWrote out file "/tmp/test_repair_verify.data"
100632024-02-13T21:43:17.922Z
100642024-02-13T21:43:17.922ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100652024-02-13T21:43:17.941ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100662024-02-13T21:43:17.941Z 0 000-019 87 87 87 157 157 157 F F F
100672024-02-13T21:43:17.941Z 1 020-039 90 90 90 166 166 166 F T T <---
100682024-02-13T21:43:17.941Z 2 040-059 88 88 88 158 158 158 F F F
100692024-02-13T21:43:17.941Z 3 060-079 88 88 88 158 158 158 F T T <---
100702024-02-13T21:43:17.941Z 4 080-099 87 87 87 157 157 157 F F F
100712024-02-13T21:43:17.941Z 5 100-119 79 79 79 136 136 136 F T T <---
100722024-02-13T21:43:17.941Z 6 120-139 90 90 90 168 168 168 F T T <---
100732024-02-13T21:43:17.941Z 7 140-159 88 88 88 162 162 162 F F F
100742024-02-13T21:43:17.941Z 8 160-179 90 90 90 165 165 165 F T T <---
100752024-02-13T21:43:17.941Z 9 180-199 90 90 90 165 165 165 F T T <---
100762024-02-13T21:43:17.942Z 10 200-219 90 90 90 166 166 166 F F F
100772024-02-13T21:43:17.942Z 11 220-239 90 90 90 166 166 166 F T T <---
100782024-02-13T21:43:17.942Z 12 240-259 89 89 89 163 163 163 F T T <---
100792024-02-13T21:43:17.942Z 13 260-279 91 91 91 169 169 169 F F F
100802024-02-13T21:43:17.942Z 14 280-299 86 86 86 152 152 152 F F F
100812024-02-13T21:43:17.942Z 15 300-319 84 84 84 148 148 148 F T T <---
100822024-02-13T21:43:17.942Z 16 320-339 88 88 88 158 158 158 F T T <---
100832024-02-13T21:43:17.942Z 17 340-359 88 88 88 158 158 158 F F F
100842024-02-13T21:43:17.942Z 18 360-379 90 90 90 166 166 166 F T T <---
100852024-02-13T21:43:17.942Z 19 380-399 90 90 90 166 166 166 F T T <---
100862024-02-13T21:43:17.942Z 20 400-419 89 89 89 163 163 163 F T T <---
100872024-02-13T21:43:17.942Z 21 420-439 86 86 86 151 151 151 F T T <---
100882024-02-13T21:43:17.942Z 22 440-459 90 90 90 164 164 164 F F F
100892024-02-13T21:43:17.942Z 23 460-479 90 90 90 166 166 166 F F F
100902024-02-13T21:43:17.942Z 24 480-499 82 82 82 145 145 145 F F F
100912024-02-13T21:43:17.942Z 25 500-519 86 86 86 156 156 156 F F F
100922024-02-13T21:43:17.942Z 26 520-539 91 91 91 169 169 169 F F F
100932024-02-13T21:43:17.942Z 27 540-559 90 90 90 167 167 167 F T T <---
100942024-02-13T21:43:17.942Z 28 560-579 91 91 91 169 169 169 F F F
100952024-02-13T21:43:17.942Z 29 580-599 86 86 86 154 154 154 F T T <---
100962024-02-13T21:43:17.942ZMax gen: 91, Max flush: 169
100972024-02-13T21:43:17.942ZError: Difference in extent metadata found!
100982024-02-13T21:43:17.942ZA Difference in extent metadata is expected here
100992024-02-13T21:43:17.942ZOn loop 46
101002024-02-13T21:43:17.943Z
101012024-02-13T21:43:17.943ZVerifying data now
101022024-02-13T21:43:22.711ZNo change after verify
101032024-02-13T21:43:22.711ZLoop: 46 Downstairs dump after verify (and repair):
101042024-02-13T21:43:22.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101052024-02-13T21:43:22.725Z 0 000-019 87 87 87 157 157 157 F F F
101062024-02-13T21:43:22.725Z 1 020-039 92 92 92 170 170 170 F F F
101072024-02-13T21:43:22.725Z 2 040-059 88 88 88 158 158 158 F F F
101082024-02-13T21:43:22.725Z 3 060-079 92 92 92 170 170 170 F F F
101092024-02-13T21:43:22.726Z 4 080-099 87 87 87 157 157 157 F F F
101102024-02-13T21:43:22.726Z 5 100-119 92 92 92 170 170 170 F F F
101112024-02-13T21:43:22.726Z 6 120-139 92 92 92 170 170 170 F F F
101122024-02-13T21:43:22.726Z 7 140-159 88 88 88 162 162 162 F F F
101132024-02-13T21:43:22.727Z 8 160-179 92 92 92 170 170 170 F F F
101142024-02-13T21:43:22.736Z 9 180-199 92 92 92 170 170 170 F F F
101152024-02-13T21:43:22.736Z 10 200-219 90 90 90 166 166 166 F F F
101162024-02-13T21:43:22.736Z 11 220-239 92 92 92 170 170 170 F F F
101172024-02-13T21:43:22.736Z 12 240-259 92 92 92 170 170 170 F F F
101182024-02-13T21:43:22.736Z 13 260-279 91 91 91 169 169 169 F F F
101192024-02-13T21:43:22.736Z 14 280-299 86 86 86 152 152 152 F F F
101202024-02-13T21:43:22.736Z 15 300-319 92 92 92 170 170 170 F F F
101212024-02-13T21:43:22.737Z 16 320-339 92 92 92 170 170 170 F F F
101222024-02-13T21:43:22.737Z 17 340-359 88 88 88 158 158 158 F F F
101232024-02-13T21:43:22.737Z 18 360-379 92 92 92 170 170 170 F F F
101242024-02-13T21:43:22.737Z 19 380-399 92 92 92 170 170 170 F F F
101252024-02-13T21:43:22.737Z 20 400-419 92 92 92 170 170 170 F F F
101262024-02-13T21:43:22.737Z 21 420-439 92 92 92 170 170 170 F F F
101272024-02-13T21:43:22.738Z 22 440-459 90 90 90 164 164 164 F F F
101282024-02-13T21:43:22.738Z 23 460-479 90 90 90 166 166 166 F F F
101292024-02-13T21:43:22.738Z 24 480-499 82 82 82 145 145 145 F F F
101302024-02-13T21:43:22.738Z 25 500-519 86 86 86 156 156 156 F F F
101312024-02-13T21:43:22.738Z 26 520-539 91 91 91 169 169 169 F F F
101322024-02-13T21:43:22.738Z 27 540-559 92 92 92 170 170 170 F F F
101332024-02-13T21:43:22.738Z 28 560-579 91 91 91 169 169 169 F F F
101342024-02-13T21:43:22.738Z 29 580-599 92 92 92 170 170 170 F F F
101352024-02-13T21:43:22.738ZMax gen: 92, Max flush: 170
101362024-02-13T21:43:22.739Z
101372024-02-13T21:43:22.739ZBegin loop 47
101382024-02-13T21:43:22.739ZDownstairs to restart: 1
101392024-02-13T21:43:22.753Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.504933758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514}
101402024-02-13T21:43:22.753Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50523266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514Crucible runtime is spawned
101412024-02-13T21:43:22.753Z}
101422024-02-13T21:43:22.753Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505277543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514}
101432024-02-13T21:43:22.754Z{"msg":"Crucible 2eaf1f83-2be6-42e8-8383-3741f0a2e457 has session id: 88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505299845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101442024-02-13T21:43:22.754Z{"msg":"Upstairs opts: Upstairs UUID: 2eaf1f83-2be6-42e8-8383-3741f0a2e457, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50534067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101452024-02-13T21:43:22.754Z{"msg":"Crucible stats registered with UUID: 2eaf1f83-2be6-42e8-8383-3741f0a2e457","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505360733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101462024-02-13T21:43:22.754Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505390183Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514}
101472024-02-13T21:43:22.754Z{"msg":"2eaf1f83-2be6-42e8-8383-3741f0a2e457 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505525993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101482024-02-13T21:43:22.754Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505578874Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101492024-02-13T21:43:22.754Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505608633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101502024-02-13T21:43:22.754Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505634204Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101512024-02-13T21:43:22.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505669011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101522024-02-13T21:43:22.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505695712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101532024-02-13T21:43:22.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50572822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101542024-02-13T21:43:22.755Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505750942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101552024-02-13T21:43:22.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505793286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101562024-02-13T21:43:22.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505835311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101572024-02-13T21:43:22.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.505861452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101582024-02-13T21:43:22.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.506008638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101592024-02-13T21:43:22.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.506061959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"io task","client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101602024-02-13T21:43:22.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.506521802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101612024-02-13T21:43:22.756Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.506568155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101622024-02-13T21:43:22.756Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.506648516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101632024-02-13T21:43:22.756Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50674681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101642024-02-13T21:43:22.770Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507183422Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101652024-02-13T21:43:22.770Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507225157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101662024-02-13T21:43:22.770Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507528337Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101672024-02-13T21:43:22.770Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507565553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101682024-02-13T21:43:22.770Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507726205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101692024-02-13T21:43:22.771Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507770499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101702024-02-13T21:43:22.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507808325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101712024-02-13T21:43:22.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507836335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101722024-02-13T21:43:22.771Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507865335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101732024-02-13T21:43:22.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.507883448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101742024-02-13T21:43:22.771Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508060904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101752024-02-13T21:43:22.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508453871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101762024-02-13T21:43:22.771Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508486889Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101772024-02-13T21:43:22.771Z{"msg":"[0]R flush_numbers[0..12]: [157, 170, 158, 170, 157, 170, 170, 162, 170, 170, 166, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508536921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101782024-02-13T21:43:22.772Z{"msg":"[0]R generation[0..12]: [87, 92, 88, 92, 87, 92, 92, 88, 92, 92, 90, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508591911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101792024-02-13T21:43:22.772Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508633516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101802024-02-13T21:43:22.772Z{"msg":"[1]R flush_numbers[0..12]: [157, 170, 158, 170, 157, 170, 170, 162, 170, 170, 166, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50864975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101812024-02-13T21:43:22.772Z{"msg":"[1]R generation[0..12]: [87, 92, 88, 92, 87, 92, 92, 88, 92, 92, 90, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508664565Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101822024-02-13T21:43:22.772ZWait for a query_work_queue command to finish before sending IO
101832024-02-13T21:43:22.773Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508690015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101842024-02-13T21:43:22.773Z{"msg":"[2]R flush_numbers[0..12]: [157, 170, 158, 170, 157, 170, 170, 162, 170, 170, 166, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.50870607Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101852024-02-13T21:43:22.773Z{"msg":"[2]R generation[0..12]: [87, 92, 88, 92, 87, 92, 92, 88, 92, 92, 90, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508726792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101862024-02-13T21:43:22.773Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508741367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101872024-02-13T21:43:22.773Z{"msg":"Max found gen is 93","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508766478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101882024-02-13T21:43:22.773Z{"msg":"Generation requested: 94 >= found:93","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508792298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101892024-02-13T21:43:22.773Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508812541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101902024-02-13T21:43:22.774Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508832624Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"":"downstairs","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101912024-02-13T21:43:22.774Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508846519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101922024-02-13T21:43:22.774Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508869101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"0","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101932024-02-13T21:43:22.774Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508884985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"1","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101942024-02-13T21:43:22.774Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508903458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"client":"2","":"downstairs","session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101952024-02-13T21:43:22.774Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508932378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101962024-02-13T21:43:22.774Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508958918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101972024-02-13T21:43:22.774Z{"msg":"2eaf1f83-2be6-42e8-8383-3741f0a2e457 is now active with session: 88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508972783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101982024-02-13T21:43:22.774Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.508985148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514,"session_id":"88e8c4ec-46c1-4cb1-808b-9f9c478f0fb1"}
101992024-02-13T21:43:22.774Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:21.509002102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514}
102002024-02-13T21:43:22.775ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
102012024-02-13T21:43:22.775ZLoading write count information from file "/tmp/test_repair_verify.data"
102022024-02-13T21:43:22.775ZRun Repair workload
102032024-02-13T21:43:22.775Z01/30 Flush
102042024-02-13T21:43:22.775Z02/30 Flush
102052024-02-13T21:43:22.775Z03/30 Write block 539 len 12288 data: 8 10 11
102062024-02-13T21:43:22.775Z04/30 Read block 394 len 4096
102072024-02-13T21:43:22.775Z05/30 Read block 535 len 20480
102082024-02-13T21:43:22.776Z06/30 Read block 318 len 20480
102092024-02-13T21:43:22.776Z07/30 Write block 188 len 40960 data: 11 12 11 10 9 9 11 10 11 12
102102024-02-13T21:43:22.776Z08/30 Flush
102112024-02-13T21:43:22.776Z09/30 Flush
102122024-02-13T21:43:22.776Z10/30 Write block 133 len 16384 data: 9 7 8 7
102132024-02-13T21:43:22.776Z11/30 Read block 304 len 24576
102142024-02-13T21:43:22.776Z12/30 Read block 94 len 36864
102152024-02-13T21:43:22.776Z13/30 Write block 325 len 32768 data: 12 12 11 9 7 6 10 11
102162024-02-13T21:43:22.776Z14/30 Read block 218 len 32768
102172024-02-13T21:43:22.776Z15/30 Write block 490 len 4096 data: 5
102182024-02-13T21:43:22.777Z16/30 Read block 586 len 4096
102192024-02-13T21:43:22.777Z17/30 Write block 185 len 4096 data: 12
102202024-02-13T21:43:22.777Z18/30 Flush
102212024-02-13T21:43:22.782Z19/30 Write block 20 len 24576 data: 9 9 10 8 8 8
102222024-02-13T21:43:22.782Z20/30 Read block 81 len 16384
102232024-02-13T21:43:22.782Z21/30 Read block 329 len 32768
102242024-02-13T21:43:22.782Z22/30 Read block 399 len 36864
102252024-02-13T21:43:22.782Z23/30 Read block 369 len 8192
102262024-02-13T21:43:22.783Z24/30 Read block 77 len 12288
102272024-02-13T21:43:22.783Z25/30 Write block 259 len 4096 data: 6
102282024-02-13T21:43:22.783Z26/30 Read block 152 len 32768
102292024-02-13T21:43:22.783Z27/30 Write block 195 len 32768 data: 11 12 13 12 12 10 10 9
102302024-02-13T21:43:22.783Z28/30 Read block 589 len 24576
102312024-02-13T21:43:22.783Z29/30 Read block 178 len 12288
102322024-02-13T21:43:22.783Z30/30 Read block 406 len 4096
102332024-02-13T21:43:22.783Z----------------------------------------------------------------
102342024-02-13T21:43:22.783Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:28
102352024-02-13T21:43:22.784ZStates: Active Active Active
102362024-02-13T21:43:22.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102372024-02-13T21:43:22.784Z 3 Acked 1002 Write 3 Done Sent Done false
102382024-02-13T21:43:22.784Z 4 Acked 1003 Read 1 Done Sent Done false
102392024-02-13T21:43:22.784Z 5 Acked 1004 Read 5 Done Sent Done false
102402024-02-13T21:43:22.784Z 6 Acked 1005 Read 5 Done Sent Done false
102412024-02-13T21:43:22.784Z 7 Acked 1006 Write 10 Done Sent Done false
102422024-02-13T21:43:22.784Z 8 Acked 1007 Flush 0 Done Sent Done false
102432024-02-13T21:43:22.784Z 9 Acked 1008 Flush 0 Done Sent Done false
102442024-02-13T21:43:22.785Z 10 Acked 1009 Write 4 Done Sent Done false
102452024-02-13T21:43:22.785Z 11 Acked 1010 Read 6 Done Sent Done false
102462024-02-13T21:43:22.785Z 12 Acked 1011 Read 9 Done Sent Done false
102472024-02-13T21:43:22.785Z 13 Acked 1012 Write 8 Done Sent Done false
102482024-02-13T21:43:22.785Z 14 Acked 1013 Read 8 Done Sent Done false
102492024-02-13T21:43:22.785Z 15 Acked 1014 Write 1 Done Sent Done false
102502024-02-13T21:43:22.785Z 16 Acked 1015 Read 1 Done Sent Done false
102512024-02-13T21:43:22.786Z 17 Acked 1016 Write 1 Done Sent Done false
102522024-02-13T21:43:22.786Z 18 Acked 1017 Flush 0 Done Sent Done false
102532024-02-13T21:43:22.786Z 19 Acked 1018 Write 6 Done Sent Done false
102542024-02-13T21:43:22.786Z 20 Acked 1019 Read 4 Done Sent Done false
102552024-02-13T21:43:22.786Z 21 Acked 1020 Read 8 Done Sent Done false
102562024-02-13T21:43:22.786Z 22 Acked 1021 Read 9 Done Sent Done false
102572024-02-13T21:43:22.787Z 23 Acked 1022 Read 2 Done Sent Done false
102582024-02-13T21:43:22.787Z 24 Acked 1023 Read 3 Done Sent Done false
102592024-02-13T21:43:22.787Z 25 Acked 1024 Write 1 Done Sent Done false
102602024-02-13T21:43:22.787Z 26 Acked 1025 Read 8 Done Sent Done false
102612024-02-13T21:43:22.787Z 27 Acked 1026 Write 8 Done Sent Done false
102622024-02-13T21:43:22.787Z 28 Acked 1027 Read 6 Done Sent Done false
102632024-02-13T21:43:22.787Z 29 Acked 1028 Read 3 Done Sent Done false
102642024-02-13T21:43:22.788Z 30 Acked 1029 Read 1 Done Sent Done false
102652024-02-13T21:43:22.788Z STATES DS:0 DS:1 DS:2 TOTAL
102662024-02-13T21:43:22.788Z New 0 0 0 0
102672024-02-13T21:43:22.788Z Sent 0 28 0 28
102682024-02-13T21:43:22.789Z Done 28 0 28 56
102692024-02-13T21:43:22.789Z Skipped 0 0 0 0
102702024-02-13T21:43:22.789Z Error 0 0 0 0
102712024-02-13T21:43:22.789ZLast Flush: 1017 1001 1017
102722024-02-13T21:43:22.789ZDownstairs last five completed: 1001 1000
102732024-02-13T21:43:22.789ZUpstairs last five completed: 30 29 28 27 26
102742024-02-13T21:43:22.789Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:21.53718412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1514}
102752024-02-13T21:43:22.789ZWrote out file "/tmp/test_repair_verify.data"
102762024-02-13T21:43:22.790Z
102772024-02-13T21:43:22.800ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102782024-02-13T21:43:22.815ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102792024-02-13T21:43:22.815Z 0 000-019 87 87 87 157 157 157 F F F
102802024-02-13T21:43:22.815Z 1 020-039 92 92 92 170 170 170 T F T <---
102812024-02-13T21:43:22.816Z 2 040-059 88 88 88 158 158 158 F F F
102822024-02-13T21:43:22.816Z 3 060-079 92 92 92 170 170 170 F F F
102832024-02-13T21:43:22.816Z 4 080-099 87 87 87 157 157 157 F F F
102842024-02-13T21:43:22.816Z 5 100-119 92 92 92 170 170 170 F F F
102852024-02-13T21:43:22.817Z 6 120-139 94 92 94 175 170 175 F F F <---
102862024-02-13T21:43:22.817Z 7 140-159 88 88 88 162 162 162 F F F
102872024-02-13T21:43:22.817Z 8 160-179 92 92 92 170 170 170 F F F
102882024-02-13T21:43:22.817Z 9 180-199 94 92 94 175 170 175 T F T <---
102892024-02-13T21:43:22.817Z 10 200-219 90 90 90 166 166 166 T F T <---
102902024-02-13T21:43:22.817Z 11 220-239 92 92 92 170 170 170 F F F
102912024-02-13T21:43:22.817Z 12 240-259 92 92 92 170 170 170 T F T <---
102922024-02-13T21:43:22.817Z 13 260-279 91 91 91 169 169 169 F F F
102932024-02-13T21:43:22.817Z 14 280-299 86 86 86 152 152 152 F F F
102942024-02-13T21:43:22.818Z 15 300-319 92 92 92 170 170 170 F F F
102952024-02-13T21:43:22.818Z 16 320-339 94 92 94 175 170 175 F F F <---
102962024-02-13T21:43:22.818Z 17 340-359 88 88 88 158 158 158 F F F
102972024-02-13T21:43:22.819Z 18 360-379 92 92 92 170 170 170 F F F
102982024-02-13T21:43:22.819Z 19 380-399 92 92 92 170 170 170 F F F
102992024-02-13T21:43:22.819Z 20 400-419 92 92 92 170 170 170 F F F
103002024-02-13T21:43:22.819Z 21 420-439 92 92 92 170 170 170 F F F
103012024-02-13T21:43:22.819Z 22 440-459 90 90 90 164 164 164 F F F
103022024-02-13T21:43:22.820Z 23 460-479 90 90 90 166 166 166 F F F
103032024-02-13T21:43:22.820Z 24 480-499 94 82 94 175 145 175 F F F <---
103042024-02-13T21:43:22.820Z 25 500-519 86 86 86 156 156 156 F F F
103052024-02-13T21:43:22.820Z 26 520-539 94 91 94 173 169 173 F F F <---
103062024-02-13T21:43:22.820Z 27 540-559 94 92 94 173 170 173 F F F <---
103072024-02-13T21:43:22.820Z 28 560-579 91 91 91 169 169 169 F F F
103082024-02-13T21:43:22.820Z 29 580-599 92 92 92 170 170 170 F F F
103092024-02-13T21:43:22.820ZMax gen: 94, Max flush: 175
103102024-02-13T21:43:22.821ZError: Difference in extent metadata found!
103112024-02-13T21:43:22.821ZA Difference in extent metadata is expected here
103122024-02-13T21:43:22.821ZOn loop 47
103132024-02-13T21:43:22.821Z
103142024-02-13T21:43:22.821ZVerifying data now
103152024-02-13T21:43:27.291ZNo change after verify
103162024-02-13T21:43:27.291ZLoop: 47 Downstairs dump after verify (and repair):
103172024-02-13T21:43:27.306ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103182024-02-13T21:43:27.306Z 0 000-019 87 87 87 157 157 157 F F F
103192024-02-13T21:43:27.306Z 1 020-039 95 95 95 176 176 176 F F F
103202024-02-13T21:43:27.306Z 2 040-059 88 88 88 158 158 158 F F F
103212024-02-13T21:43:27.306Z 3 060-079 92 92 92 170 170 170 F F F
103222024-02-13T21:43:27.306Z 4 080-099 87 87 87 157 157 157 F F F
103232024-02-13T21:43:27.306Z 5 100-119 92 92 92 170 170 170 F F F
103242024-02-13T21:43:27.306Z 6 120-139 94 94 94 175 175 175 F F F
103252024-02-13T21:43:27.306Z 7 140-159 88 88 88 162 162 162 F F F
103262024-02-13T21:43:27.306Z 8 160-179 92 92 92 170 170 170 F F F
103272024-02-13T21:43:27.306Z 9 180-199 95 95 95 176 176 176 F F F
103282024-02-13T21:43:27.306Z 10 200-219 95 95 95 176 176 176 F F F
103292024-02-13T21:43:27.306Z 11 220-239 92 92 92 170 170 170 F F F
103302024-02-13T21:43:27.306Z 12 240-259 95 95 95 176 176 176 F F F
103312024-02-13T21:43:27.310Z 13 260-279 91 91 91 169 169 169 F F F
103322024-02-13T21:43:27.310Z 14 280-299 86 86 86 152 152 152 F F F
103332024-02-13T21:43:27.310Z 15 300-319 92 92 92 170 170 170 F F F
103342024-02-13T21:43:27.310Z 16 320-339 94 94 94 175 175 175 F F F
103352024-02-13T21:43:27.310Z 17 340-359 88 88 88 158 158 158 F F F
103362024-02-13T21:43:27.310Z 18 360-379 92 92 92 170 170 170 F F F
103372024-02-13T21:43:27.310Z 19 380-399 92 92 92 170 170 170 F F F
103382024-02-13T21:43:27.310Z 20 400-419 92 92 92 170 170 170 F F F
103392024-02-13T21:43:27.310Z 21 420-439 92 92 92 170 170 170 F F F
103402024-02-13T21:43:27.310Z 22 440-459 90 90 90 164 164 164 F F F
103412024-02-13T21:43:27.310Z 23 460-479 90 90 90 166 166 166 F F F
103422024-02-13T21:43:27.310Z 24 480-499 94 94 94 175 175 175 F F F
103432024-02-13T21:43:27.310Z 25 500-519 86 86 86 156 156 156 F F F
103442024-02-13T21:43:27.310Z 26 520-539 94 94 94 173 173 173 F F F
103452024-02-13T21:43:27.310Z 27 540-559 94 94 94 173 173 173 F F F
103462024-02-13T21:43:27.310Z 28 560-579 91 91 91 169 169 169 F F F
103472024-02-13T21:43:27.310Z 29 580-599 92 92 92 170 170 170 F F F
103482024-02-13T21:43:27.310ZMax gen: 95, Max flush: 176
103492024-02-13T21:43:27.311Z
103502024-02-13T21:43:27.311ZBegin loop 48
103512024-02-13T21:43:27.311ZDownstairs to restart: 2
103522024-02-13T21:43:27.327Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087375731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
103532024-02-13T21:43:27.327Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087660498Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
103542024-02-13T21:43:27.327Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
103552024-02-13T21:43:27.327Z,"time":"2024-02-13T21:43:26.087687088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
103562024-02-13T21:43:27.331Z{"msg":"Crucible 5f41f6b3-b486-45a3-a146-09befdda7616 has session id: 228560d2-5a91-401a-aa07-e400f94abaa6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087723575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103572024-02-13T21:43:27.331Z{"msg":"Upstairs opts: Upstairs UUID: 5f41f6b3-b486-45a3-a146-09befdda7616, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087757043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103582024-02-13T21:43:27.331Z{"msg":"Crucible stats registered with UUID: 5f41f6b3-b486-45a3-a146-09befdda7616","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087799747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103592024-02-13T21:43:27.331Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087835464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
103602024-02-13T21:43:27.331Z{"msg":"5f41f6b3-b486-45a3-a146-09befdda7616 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087917724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103612024-02-13T21:43:27.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.087942216Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103622024-02-13T21:43:27.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.08795765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103632024-02-13T21:43:27.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.08798469Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103642024-02-13T21:43:27.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088006702Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103652024-02-13T21:43:27.332Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088030364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103662024-02-13T21:43:27.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088055375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103672024-02-13T21:43:27.332Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088072948Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103682024-02-13T21:43:27.332Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088112454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103692024-02-13T21:43:27.332Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088141953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103702024-02-13T21:43:27.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088172052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103712024-02-13T21:43:27.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088265468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103722024-02-13T21:43:27.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088300996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"io task","client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103732024-02-13T21:43:27.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088920531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103742024-02-13T21:43:27.332Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.088961676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103752024-02-13T21:43:27.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.08897862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103762024-02-13T21:43:27.332Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.089014297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103772024-02-13T21:43:27.332Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.089558009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103782024-02-13T21:43:27.332Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.089600274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103792024-02-13T21:43:27.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090030858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103802024-02-13T21:43:27.332Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090117067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103812024-02-13T21:43:27.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090149445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103822024-02-13T21:43:27.332Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090184442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103832024-02-13T21:43:27.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090206185Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103842024-02-13T21:43:27.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090228146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103852024-02-13T21:43:27.332Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090242501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103862024-02-13T21:43:27.333Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090263204Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103872024-02-13T21:43:27.333Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090534026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103882024-02-13T21:43:27.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.090977135Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103892024-02-13T21:43:27.338Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091030825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103902024-02-13T21:43:27.338Z{"msg":"[0]R flush_numbers[0..12]: [157, 176, 158, 170, 157, 170, 175, 162, 170, 176, 176, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091050288Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103912024-02-13T21:43:27.338Z{"msg":"[0]R generation[0..12]: [87, 95, 88, 92, 87, 92, 94, 88, 92, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091078018Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103922024-02-13T21:43:27.338Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091093672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103932024-02-13T21:43:27.338Z{"msg":"[1]R flush_numbers[0..12]: [157, 176, 158, 170, 157, 170, 175, 162, 170, 176, 176, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091115105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
103942024-02-13T21:43:27.338Zdownstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103952024-02-13T21:43:27.338Z{"msg":"[1]R generation[0..12]: [87, 95, 88, 92, 87, 92, 94, 88, 92, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091144194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103962024-02-13T21:43:27.338Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091171424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103972024-02-13T21:43:27.338Z{"msg":"[2]R flush_numbers[0..12]: [157, 176, 158, 170, 157, 170, 175, 162, 170, 176, 176, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091186319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103982024-02-13T21:43:27.338Z{"msg":"[2]R generation[0..12]: [87, 95, 88, 92, 87, 92, 94, 88, 92, 95, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091207171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
103992024-02-13T21:43:27.338Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091221446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104002024-02-13T21:43:27.338Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091234941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104012024-02-13T21:43:27.338Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091259622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104022024-02-13T21:43:27.338Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091279255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104032024-02-13T21:43:27.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091301227Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"":"downstairs","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104042024-02-13T21:43:27.338Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091334195Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104052024-02-13T21:43:27.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091357117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"0","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104062024-02-13T21:43:27.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091380778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"1","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104072024-02-13T21:43:27.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091412377Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"client":"2","":"downstairs","session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104082024-02-13T21:43:27.338Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091437878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104092024-02-13T21:43:27.338Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091477563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104102024-02-13T21:43:27.339Z{"msg":"5f41f6b3-b486-45a3-a146-09befdda7616 is now active with session: 228560d2-5a91-401a-aa07-e400f94abaa6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091502794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104112024-02-13T21:43:27.339Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091528965Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522,"session_id":"228560d2-5a91-401a-aa07-e400f94abaa6"}
104122024-02-13T21:43:27.339Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:26.091546388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
104132024-02-13T21:43:27.339ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104142024-02-13T21:43:27.339ZLoading write count information from file "/tmp/test_repair_verify.data"
104152024-02-13T21:43:27.339ZRun Repair workload
104162024-02-13T21:43:27.339Z01/30 Write block 92 len 24576 data: 6 6 7 7 8 9
104172024-02-13T21:43:27.339Z02/30 Write block 348 len 16384 data: 8 6 6 8
104182024-02-13T21:43:27.339Z03/30 Write block 272 len 40960 data: 8 8 9 10 11 13 11 12 10 6
104192024-02-13T21:43:27.339Z04/30 Read block 277 len 12288
104202024-02-13T21:43:27.339Z05/30 Write block 424 len 36864 data: 8 8 9 8 9 11 9 10 12
104212024-02-13T21:43:27.342Z06/30 Flush
104222024-02-13T21:43:27.342Z07/30 Write block 44 len 16384 data: 5 5 6 5
104232024-02-13T21:43:27.342Z08/30 Read block 91 len 32768
104242024-02-13T21:43:27.342Z09/30 Flush
104252024-02-13T21:43:27.345Z10/30 Write block 225 len 40960 data: 7 6 7 6 5 6 6 5 5 7
104262024-02-13T21:43:27.345Z11/30 Read block 476 len 12288
104272024-02-13T21:43:27.348Z12/30 Read block 505 len 24576
104282024-02-13T21:43:27.348Z13/30 Write block 248 len 24576 data: 4 5 5 7 6 6
104292024-02-13T21:43:27.348Z14/30 Read block 419 len 32768
104302024-02-13T21:43:27.351Z15/30 Read block 551 len 36864
104312024-02-13T21:43:27.352Z16/30 Write block 474 len 20480 data: 11 10 7 6 8
104322024-02-13T21:43:27.352Z17/30 Read block 154 len 20480
104332024-02-13T21:43:27.352Z18/30 Read block 96 len 32768
104342024-02-13T21:43:27.355Z19/30 Write block 363 len 40960 data: 8 7 6 6 9 8 8 8 10 10
104352024-02-13T21:43:27.355Z20/30 Write block 504 len 36864 data: 7 7 9 9 9 10 11 11 10
104362024-02-13T21:43:27.355Z21/30 Read block 148 len 36864
104372024-02-13T21:43:27.358Z22/30 Read block 480 len 32768
104382024-02-13T21:43:27.358Z23/30 Flush
104392024-02-13T21:43:27.361Z24/30 Write block 10 len 12288 data: 10 12 10
104402024-02-13T21:43:27.361Z25/30 Read block 468 len 28672
104412024-02-13T21:43:27.361Z26/30 Flush
104422024-02-13T21:43:27.364Z27/30 Write block 322 len 4096 data: 10
104432024-02-13T21:43:27.364Z28/30 Write block 13 len 12288 data: 12 13 13
104442024-02-13T21:43:27.364Z29/30 Read block 469 len 8192
104452024-02-13T21:43:27.368Z30/30 Write block 59 len 8192 data: 11 9
104462024-02-13T21:43:27.368Z----------------------------------------------------------------
104472024-02-13T21:43:27.368Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
104482024-02-13T21:43:27.368ZStates: Active Active Active
104492024-02-13T21:43:27.368ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104502024-02-13T21:43:27.368Z 1 Acked 1000 Write 6 Done Done Done false
104512024-02-13T21:43:27.368Z 2 Acked 1001 Write 4 Done Done Sent false
104522024-02-13T21:43:27.368Z 3 Acked 1002 Write 10 Done Done Sent false
104532024-02-13T21:43:27.368Z 4 Acked 1003 Read 3 Done Done Sent false
104542024-02-13T21:43:27.368Z 5 Acked 1004 Write 9 Done Done Sent false
104552024-02-13T21:43:27.368Z 6 Acked 1005 Flush 0 Done Done Sent false
104562024-02-13T21:43:27.368Z 7 Acked 1006 Write 4 Done Done Sent false
104572024-02-13T21:43:27.368Z 8 Acked 1007 Read 8 Done Done Sent false
104582024-02-13T21:43:27.368Z 9 Acked 1008 Flush 0 Done Done Sent false
104592024-02-13T21:43:27.368Z 10 Acked 1009 Write 10 Done Done Sent false
104602024-02-13T21:43:27.368Z 11 Acked 1010 Read 3 Done Done Sent false
104612024-02-13T21:43:27.368Z 12 Acked 1011 Read 6 Done Done Sent false
104622024-02-13T21:43:27.368Z 13 Acked 1012 Write 6 Done Done Sent false
104632024-02-13T21:43:27.368Z 14 Acked 1013 Read 8 Done Done Sent false
104642024-02-13T21:43:27.368Z 15 Acked 1014 Read 9 Done Done Sent false
104652024-02-13T21:43:27.368Z 16 Acked 1015 Write 5 Done Done Sent false
104662024-02-13T21:43:27.368Z 17 Acked 1016 Read 5 Done Done Sent false
104672024-02-13T21:43:27.368Z 18 Acked 1017 Read 8 Done Done Sent false
104682024-02-13T21:43:27.369Z 19 Acked 1018 Write 10 Done Done Sent false
104692024-02-13T21:43:27.369Z 20 Acked 1019 Write 9 Done Done Sent false
104702024-02-13T21:43:27.369Z 21 Acked 1020 Read 9 Done Done Sent false
104712024-02-13T21:43:27.369Z 22 Acked 1021 Read 8 Done Done Sent false
104722024-02-13T21:43:27.369Z 23 Acked 1022 Flush 0 Done Done Sent false
104732024-02-13T21:43:27.369Z 24 Acked 1023 Write 3 Done Done Sent false
104742024-02-13T21:43:27.369Z 25 Acked 1024 Read 7 Done Done Sent false
104752024-02-13T21:43:27.369Z 26 Acked 1025 Flush 0 Done Done Sent false
104762024-02-13T21:43:27.369Z 27 Acked 1026 Write 1 Done Done Sent false
104772024-02-13T21:43:27.369Z 28 Acked 1027 Write 3 Done Done Sent false
104782024-02-13T21:43:27.369Z 29 Acked 1028 Read 2 Done Done Sent false
104792024-02-13T21:43:27.369Z 30 Acked 1029 Write 2 Sent Sent Sent false
104802024-02-13T21:43:27.369Z STATES DS:0 DS:1 DS:2 TOTAL
104812024-02-13T21:43:27.369Z New 0 0 0 0
104822024-02-13T21:43:27.369Z Sent 1 1 29 31
104832024-02-13T21:43:27.369Z Done 29 29 1 59
104842024-02-13T21:43:27.369Z Skipped 0 0 0 0
104852024-02-13T21:43:27.369Z Error 0 0 0 0
104862024-02-13T21:43:27.369ZLast Flush: 1025 1025 0
104872024-02-13T21:43:27.369ZDownstairs last five completed:
104882024-02-13T21:43:27.369ZUpstairs last five completed: 30 29 28 27 26
104892024-02-13T21:43:27.369Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:26.12627641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1522}
104902024-02-13T21:43:27.369ZWrote out file "/tmp/test_repair_verify.data"
104912024-02-13T21:43:27.373Z
104922024-02-13T21:43:27.373ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104932024-02-13T21:43:27.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104942024-02-13T21:43:27.388Z 0 000-019 96 96 87 180 180 157 T T F <---
104952024-02-13T21:43:27.388Z 1 020-039 95 95 95 176 176 176 F F F
104962024-02-13T21:43:27.388Z 2 040-059 96 96 88 178 178 158 T T F <---
104972024-02-13T21:43:27.388Z 3 060-079 92 92 92 170 170 170 T T F <---
104982024-02-13T21:43:27.388Z 4 080-099 96 96 87 177 177 157 F F T <---
104992024-02-13T21:43:27.388Z 5 100-119 92 92 92 170 170 170 F F F
105002024-02-13T21:43:27.392Z 6 120-139 94 94 94 175 175 175 F F F
105012024-02-13T21:43:27.392Z 7 140-159 88 88 88 162 162 162 F F F
105022024-02-13T21:43:27.392Z 8 160-179 92 92 92 170 170 170 F F F
105032024-02-13T21:43:27.392Z 9 180-199 95 95 95 176 176 176 F F F
105042024-02-13T21:43:27.392Z 10 200-219 95 95 95 176 176 176 F F F
105052024-02-13T21:43:27.392Z 11 220-239 96 96 92 179 179 170 F F F <---
105062024-02-13T21:43:27.392Z 12 240-259 96 96 95 179 179 176 F F F <---
105072024-02-13T21:43:27.392Z 13 260-279 96 96 91 177 177 169 F F F <---
105082024-02-13T21:43:27.392Z 14 280-299 96 96 86 177 177 152 F F F <---
105092024-02-13T21:43:27.392Z 15 300-319 92 92 92 170 170 170 F F F
105102024-02-13T21:43:27.392Z 16 320-339 94 94 94 175 175 175 T T F <---
105112024-02-13T21:43:27.392Z 17 340-359 96 96 88 177 177 158 F F F <---
105122024-02-13T21:43:27.392Z 18 360-379 96 96 92 179 179 170 F F F <---
105132024-02-13T21:43:27.392Z 19 380-399 92 92 92 170 170 170 F F F
105142024-02-13T21:43:27.392Z 20 400-419 92 92 92 170 170 170 F F F
105152024-02-13T21:43:27.392Z 21 420-439 96 96 92 177 177 170 F F F <---
105162024-02-13T21:43:27.392Z 22 440-459 90 90 90 164 164 164 F F F
105172024-02-13T21:43:27.392Z 23 460-479 96 96 90 179 179 166 F F F <---
105182024-02-13T21:43:27.392Z 24 480-499 94 94 94 175 175 175 F F F
105192024-02-13T21:43:27.392Z 25 500-519 96 96 86 179 179 156 F F F <---
105202024-02-13T21:43:27.392Z 26 520-539 94 94 94 173 173 173 F F F
105212024-02-13T21:43:27.392Z 27 540-559 94 94 94 173 173 173 F F F
105222024-02-13T21:43:27.393Z 28 560-579 91 91 91 169 169 169 F F F
105232024-02-13T21:43:27.393Z 29 580-599 92 92 92 170 170 170 F F F
105242024-02-13T21:43:27.393ZMax gen: 96, Max flush: 180
105252024-02-13T21:43:27.393ZError: Difference in extent metadata found!
105262024-02-13T21:43:27.393ZA Difference in extent metadata is expected here
105272024-02-13T21:43:27.393ZOn loop 48
105282024-02-13T21:43:27.393Z
105292024-02-13T21:43:27.409ZVerifying data now
105302024-02-13T21:43:32.076ZNo change after verify
105312024-02-13T21:43:32.077ZLoop: 48 Downstairs dump after verify (and repair):
105322024-02-13T21:43:32.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105332024-02-13T21:43:32.091Z 0 000-019 97 97 97 181 181 181 F F F
105342024-02-13T21:43:32.091Z 1 020-039 95 95 95 176 176 176 F F F
105352024-02-13T21:43:32.091Z 2 040-059 97 97 97 181 181 181 F F F
105362024-02-13T21:43:32.091Z 3 060-079 97 97 97 181 181 181 F F F
105372024-02-13T21:43:32.091Z 4 080-099 96 96 96 177 177 177 F F F
105382024-02-13T21:43:32.091Z 5 100-119 92 92 92 170 170 170 F F F
105392024-02-13T21:43:32.097Z 6 120-139 94 94 94 175 175 175 F F F
105402024-02-13T21:43:32.097Z 7 140-159 88 88 88 162 162 162 F F F
105412024-02-13T21:43:32.097Z 8 160-179 92 92 92 170 170 170 F F F
105422024-02-13T21:43:32.097Z 9 180-199 95 95 95 176 176 176 F F F
105432024-02-13T21:43:32.097Z 10 200-219 95 95 95 176 176 176 F F F
105442024-02-13T21:43:32.097Z 11 220-239 96 96 96 179 179 179 F F F
105452024-02-13T21:43:32.097Z 12 240-259 96 96 96 179 179 179 F F F
105462024-02-13T21:43:32.097Z 13 260-279 96 96 96 177 177 177 F F F
105472024-02-13T21:43:32.097Z 14 280-299 96 96 96 177 177 177 F F F
105482024-02-13T21:43:32.097Z 15 300-319 92 92 92 170 170 170 F F F
105492024-02-13T21:43:32.097Z 16 320-339 97 97 97 181 181 181 F F F
105502024-02-13T21:43:32.097Z 17 340-359 96 96 96 177 177 177 F F F
105512024-02-13T21:43:32.097Z 18 360-379 96 96 96 179 179 179 F F F
105522024-02-13T21:43:32.097Z 19 380-399 92 92 92 170 170 170 F F F
105532024-02-13T21:43:32.097Z 20 400-419 92 92 92 170 170 170 F F F
105542024-02-13T21:43:32.097Z 21 420-439 96 96 96 177 177 177 F F F
105552024-02-13T21:43:32.097Z 22 440-459 90 90 90 164 164 164 F F F
105562024-02-13T21:43:32.097Z 23 460-479 96 96 96 179 179 179 F F F
105572024-02-13T21:43:32.097Z 24 480-499 94 94 94 175 175 175 F F F
105582024-02-13T21:43:32.098Z 25 500-519 96 96 96 179 179 179 F F F
105592024-02-13T21:43:32.098Z 26 520-539 94 94 94 173 173 173 F F F
105602024-02-13T21:43:32.098Z 27 540-559 94 94 94 173 173 173 F F F
105612024-02-13T21:43:32.098Z 28 560-579 91 91 91 169 169 169 F F F
105622024-02-13T21:43:32.098Z 29 580-599 92 92 92 170 170 170 F F F
105632024-02-13T21:43:32.098ZMax gen: 97, Max flush: 181
105642024-02-13T21:43:32.098Z
105652024-02-13T21:43:32.098ZBegin loop 49
105662024-02-13T21:43:32.098ZDownstairs to restart: 0
105672024-02-13T21:43:32.111Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869276393Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
105682024-02-13T21:43:32.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869637063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
105692024-02-13T21:43:32.125Z{Crucible runtime is spawned
105702024-02-13T21:43:32.125Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869744834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
105712024-02-13T21:43:32.125Z{"msg":"Crucible 6a4ee2ea-7328-4847-b359-64e8f110eeda has session id: 5ef57b58-9e3a-4ffd-a3a8-64b98290ab31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869788158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105722024-02-13T21:43:32.125Z{"msg":"Upstairs opts: Upstairs UUID: 6a4ee2ea-7328-4847-b359-64e8f110eeda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869817407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105732024-02-13T21:43:32.125Z{"msg":"Crucible stats registered with UUID: 6a4ee2ea-7328-4847-b359-64e8f110eeda","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869845567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105742024-02-13T21:43:32.125Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869859742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
105752024-02-13T21:43:32.126Z{"msg":"6a4ee2ea-7328-4847-b359-64e8f110eeda active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869957037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105762024-02-13T21:43:32.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.869983847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105772024-02-13T21:43:32.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870012317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105782024-02-13T21:43:32.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870036658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105792024-02-13T21:43:32.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870051052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105802024-02-13T21:43:32.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870067677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105812024-02-13T21:43:32.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870102554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105822024-02-13T21:43:32.126Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870127805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105832024-02-13T21:43:32.126Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870164971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105842024-02-13T21:43:32.126Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870188323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105852024-02-13T21:43:32.126Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870373466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105862024-02-13T21:43:32.126Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870410592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105872024-02-13T21:43:32.126Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.870437132Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"io task","client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105882024-02-13T21:43:32.126Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871236073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105892024-02-13T21:43:32.126Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871277248Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105902024-02-13T21:43:32.126Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871316433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105912024-02-13T21:43:32.126Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871344353Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105922024-02-13T21:43:32.126Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871643015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105932024-02-13T21:43:32.126Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871725305Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105942024-02-13T21:43:32.126Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.871871512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105952024-02-13T21:43:32.126Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.8719037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105962024-02-13T21:43:32.126Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872277385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105972024-02-13T21:43:32.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872450852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105982024-02-13T21:43:32.127Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872486129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
105992024-02-13T21:43:32.127Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872588022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106002024-02-13T21:43:32.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.87262149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106012024-02-13T21:43:32.127Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872652539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106022024-02-13T21:43:32.127Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872686416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106032024-02-13T21:43:32.127Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872765078Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106042024-02-13T21:43:32.127Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872801555Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106052024-02-13T21:43:32.127Z{"msg":"[0]R flush_numbers[0..12]: [181, 176, 181, 181, 177, 170, 175, 162, 170, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872820578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106062024-02-13T21:43:32.127Z{"msg":"[0]R generation[0..12]: [97, 95, 97, 97, 96, 92, 94, 88, 92, 95, 95, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872851357Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106072024-02-13T21:43:32.127Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.87286821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106082024-02-13T21:43:32.127Z{"msg":"[1]R flush_numbers[0..12]: [181, 176, 181, 181, 177, 170, 175, 162, 170, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872900439Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106092024-02-13T21:43:32.127Z{"msg":"[1]R generation[0..12]: [97, 95, 97, 97, 96, 92, 94, 88, 92, 95, 95, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872917233Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106102024-02-13T21:43:32.127Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872947802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106112024-02-13T21:43:32.127Z{"msg":"[2]R flush_numbers[0..12]: [181, 176, 181, 181, 177, 170, 175, 162, 170, 176, 176, 179]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.872963696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106122024-02-13T21:43:32.127Z{"msg":"[2]R generation[0..12]: [97, 95, 97, 97, 96, 92, 94, 88, 92, 95, 95, 96]","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO
106132024-02-13T21:43:32.127Z:"2024-02-13T21:43:30.872994615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106142024-02-13T21:43:32.127Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873013758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106152024-02-13T21:43:32.127Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873027283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106162024-02-13T21:43:32.127Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873041568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106172024-02-13T21:43:32.127Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.87306232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106182024-02-13T21:43:32.127Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873076045Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"":"downstairs","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106192024-02-13T21:43:32.127Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873092339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106202024-02-13T21:43:32.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873115501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"0","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106212024-02-13T21:43:32.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873129996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"1","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106222024-02-13T21:43:32.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873143411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"client":"2","":"downstairs","session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106232024-02-13T21:43:32.128Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873163374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106242024-02-13T21:43:32.128Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873176669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106252024-02-13T21:43:32.128Z{"msg":"6a4ee2ea-7328-4847-b359-64e8f110eeda is now active with session: 5ef57b58-9e3a-4ffd-a3a8-64b98290ab31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873189814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106262024-02-13T21:43:32.128Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873208017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530,"session_id":"5ef57b58-9e3a-4ffd-a3a8-64b98290ab31"}
106272024-02-13T21:43:32.128Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:30.873226181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
106282024-02-13T21:43:32.128ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106292024-02-13T21:43:32.128ZLoading write count information from file "/tmp/test_repair_verify.data"
106302024-02-13T21:43:32.128ZRun Repair workload
106312024-02-13T21:43:32.128Z01/30 Write block 122 len 36864 data: 8 7 6 11 11 10 8 8 5
106322024-02-13T21:43:32.135Z02/30 Write block 367 len 28672 data: 10 9 9 9 11 11 10
106332024-02-13T21:43:32.135Z03/30 Write block 542 len 32768 data: 9 9 9 11 10 9 6 8
106342024-02-13T21:43:32.135Z04/30 Read block 238 len 28672
106352024-02-13T21:43:32.135Z05/30 Read block 193 len 16384
106362024-02-13T21:43:32.135Z06/30 Read block 183 len 40960
106372024-02-13T21:43:32.135Z07/30 Write block 516 len 24576 data: 10 13 14 17 18 14
106382024-02-13T21:43:32.135Z08/30 Read block 528 len 32768
106392024-02-13T21:43:32.135Z09/30 Read block 345 len 16384
106402024-02-13T21:43:32.135Z10/30 Write block 520 len 12288 data: 19 15 13
106412024-02-13T21:43:32.135Z11/30 Write block 352 len 20480 data: 9 9 8 9 11
106422024-02-13T21:43:32.136Z12/30 Read block 285 len 28672
106432024-02-13T21:43:32.136Z13/30 Read block 368 len 12288
106442024-02-13T21:43:32.136Z14/30 Write block 159 len 32768 data: 7 6 7 7 7 5 3 4
106452024-02-13T21:43:32.136Z15/30 Read block 570 len 36864
106462024-02-13T21:43:32.136Z16/30 Write block 82 len 12288 data: 8 8 7
106472024-02-13T21:43:32.136Z17/30 Write block 108 len 40960 data: 8 7 4 7 7 7 6 6 8 9
106482024-02-13T21:43:32.136Z18/30 Write block 219 len 32768 data: 11 9 9 10 10 8 8 7
106492024-02-13T21:43:32.136Z19/30 Read block 137 len 20480
106502024-02-13T21:43:32.152Z20/30 Write block 223 len 40960 data: 11 9 9 8 8 7 6 7 7 6
106512024-02-13T21:43:32.152Z21/30 Write block 481 len 24576 data: 8 9 7 6 6 6
106522024-02-13T21:43:32.152Z22/30 Write block 502 len 8192 data: 6 5
106532024-02-13T21:43:32.152Z23/30 Write block 184 len 4096 data: 10
106542024-02-13T21:43:32.152Z24/30 Write block 281 len 24576 data: 7 5 5 5 6 5
106552024-02-13T21:43:32.152Z25/30 Write block 123 len 28672 data: 8 7 12 12 11 9 9
106562024-02-13T21:43:32.152Z26/30 Read block 106 len 12288
106572024-02-13T21:43:32.152Z27/30 Write block 584 len 36864 data: 7 8 8 7 6 7 8 7 6
106582024-02-13T21:43:32.152Z28/30 Write block 145 len 12288 data: 10 7 7
106592024-02-13T21:43:32.152Z29/30 Write block 196 len 24576 data: 13 14 13 13 11 11
106602024-02-13T21:43:32.152Z30/30 Write block 317 len 32768 data: 9 9 11 10 10 11 12 11
106612024-02-13T21:43:32.153Z----------------------------------------------------------------
106622024-02-13T21:43:32.153Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
106632024-02-13T21:43:32.153ZStates: Active Active Active
106642024-02-13T21:43:32.153ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106652024-02-13T21:43:32.153Z 1 Acked 1000 Write 9 Sent Done Done false
106662024-02-13T21:43:32.153Z 2 Acked 1001 Write 7 Sent Done Done false
106672024-02-13T21:43:32.153Z 3 Acked 1002 Write 8 Sent Done Done false
106682024-02-13T21:43:32.153Z 4 Acked 1003 Read 7 Sent Done Done false
106692024-02-13T21:43:32.153Z 5 Acked 1004 Read 4 Sent Done Done false
106702024-02-13T21:43:32.153Z 6 Acked 1005 Read 10 Sent Done Done false
106712024-02-13T21:43:32.153Z 7 Acked 1006 Write 6 Sent Done Done false
106722024-02-13T21:43:32.153Z 8 Acked 1007 Read 8 Sent Done Done false
106732024-02-13T21:43:32.153Z 9 Acked 1008 Read 4 Sent Done Done false
106742024-02-13T21:43:32.153Z 10 Acked 1009 Write 3 Sent Done Done false
106752024-02-13T21:43:32.153Z 11 Acked 1010 Write 5 Sent Done Done false
106762024-02-13T21:43:32.153Z 12 Acked 1011 Read 7 Sent Done Done false
106772024-02-13T21:43:32.153Z 13 Acked 1012 Read 3 Sent Done Done false
106782024-02-13T21:43:32.153Z 14 Acked 1013 Write 8 Sent Done Done false
106792024-02-13T21:43:32.153Z 15 Acked 1014 Read 9 Sent Done Done false
106802024-02-13T21:43:32.153Z 16 Acked 1015 Write 3 Sent Done Done false
106812024-02-13T21:43:32.153Z 17 Acked 1016 Write 10 Sent Done Done false
106822024-02-13T21:43:32.153Z 18 Acked 1017 Write 8 Sent Done Done false
106832024-02-13T21:43:32.153Z 19 Acked 1018 Read 5 Sent Done Done false
106842024-02-13T21:43:32.154Z 20 Acked 1019 Write 10 Sent Done Done false
106852024-02-13T21:43:32.154Z 21 Acked 1020 Write 6 Sent Done Done false
106862024-02-13T21:43:32.154Z 22 Acked 1021 Write 2 Sent Done Done false
106872024-02-13T21:43:32.154Z 23 Acked 1022 Write 1 Sent Done Done false
106882024-02-13T21:43:32.154Z 24 Acked 1023 Write 6 Sent Done Done false
106892024-02-13T21:43:32.154Z 25 Acked 1024 Write 7 Sent Done Done false
106902024-02-13T21:43:32.154Z 26 Acked 1025 Read 3 Sent Done Done false
106912024-02-13T21:43:32.154Z 27 Acked 1026 Write 9 Sent Done Done false
106922024-02-13T21:43:32.154Z 28 Acked 1027 Write 3 Sent Done Done false
106932024-02-13T21:43:32.154Z 29 Acked 1028 Write 6 Sent Done Done false
106942024-02-13T21:43:32.154Z 30 Acked 1029 Write 8 Sent Sent Sent false
106952024-02-13T21:43:32.154Z STATES DS:0 DS:1 DS:2 TOTAL
106962024-02-13T21:43:32.154Z New 0 0 0 0
106972024-02-13T21:43:32.154Z Sent 30 1 1 32
106982024-02-13T21:43:32.154Z Done 0 29 29 58
106992024-02-13T21:43:32.154Z Skipped 0 0 0 0
107002024-02-13T21:43:32.154Z Error 0 0 0 0
107012024-02-13T21:43:32.154ZLast Flush: 0 0 0
107022024-02-13T21:43:32.154ZDownstairs last five completed:
107032024-02-13T21:43:32.154ZUpstairs last five completed: 30 29 28 27 26
107042024-02-13T21:43:32.154Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:30.900622633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1530}
107052024-02-13T21:43:32.154ZWrote out file "/tmp/test_repair_verify.data"
107062024-02-13T21:43:32.154Z
107072024-02-13T21:43:32.154ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
107082024-02-13T21:43:32.171ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107092024-02-13T21:43:32.171Z 0 000-019 97 97 97 181 181 181 F F F
107102024-02-13T21:43:32.171Z 1 020-039 95 95 95 176 176 176 F F F
107112024-02-13T21:43:32.171Z 2 040-059 97 97 97 181 181 181 F F F
107122024-02-13T21:43:32.171Z 3 060-079 97 97 97 181 181 181 F F F
107132024-02-13T21:43:32.172Z 4 080-099 96 96 96 177 177 177 F T T <---
107142024-02-13T21:43:32.172Z 5 100-119 92 92 92 170 170 170 F T T <---
107152024-02-13T21:43:32.172Z 6 120-139 94 94 94 175 175 175 F T T <---
107162024-02-13T21:43:32.172Z 7 140-159 88 88 88 162 162 162 F T T <---
107172024-02-13T21:43:32.172Z 8 160-179 92 92 92 170 170 170 F T T <---
107182024-02-13T21:43:32.172Z 9 180-199 95 95 95 176 176 176 F T T <---
107192024-02-13T21:43:32.177Z 10 200-219 95 95 95 176 176 176 F T T <---
107202024-02-13T21:43:32.177Z 11 220-239 96 96 96 179 179 179 F T T <---
107212024-02-13T21:43:32.177Z 12 240-259 96 96 96 179 179 179 F F F
107222024-02-13T21:43:32.177Z 13 260-279 96 96 96 177 177 177 F F F
107232024-02-13T21:43:32.177Z 14 280-299 96 96 96 177 177 177 F T T <---
107242024-02-13T21:43:32.177Z 15 300-319 92 92 92 170 170 170 F T T <---
107252024-02-13T21:43:32.177Z 16 320-339 97 97 97 181 181 181 F T T <---
107262024-02-13T21:43:32.177Z 17 340-359 96 96 96 177 177 177 F T T <---
107272024-02-13T21:43:32.177Z 18 360-379 96 96 96 179 179 179 F T T <---
107282024-02-13T21:43:32.177Z 19 380-399 92 92 92 170 170 170 F F F
107292024-02-13T21:43:32.177Z 20 400-419 92 92 92 170 170 170 F F F
107302024-02-13T21:43:32.177Z 21 420-439 96 96 96 177 177 177 F F F
107312024-02-13T21:43:32.178Z 22 440-459 90 90 90 164 164 164 F F F
107322024-02-13T21:43:32.178Z 23 460-479 96 96 96 179 179 179 F F F
107332024-02-13T21:43:32.178Z 24 480-499 94 94 94 175 175 175 F T T <---
107342024-02-13T21:43:32.178Z 25 500-519 96 96 96 179 179 179 F T T <---
107352024-02-13T21:43:32.178Z 26 520-539 94 94 94 173 173 173 F T T <---
107362024-02-13T21:43:32.178Z 27 540-559 94 94 94 173 173 173 F T T <---
107372024-02-13T21:43:32.178Z 28 560-579 91 91 91 169 169 169 F F F
107382024-02-13T21:43:32.178Z 29 580-599 92 92 92 170 170 170 F T T <---
107392024-02-13T21:43:32.178ZMax gen: 97, Max flush: 181
107402024-02-13T21:43:32.178ZError: Difference in extent metadata found!
107412024-02-13T21:43:32.178ZA Difference in extent metadata is expected here
107422024-02-13T21:43:32.178ZOn loop 49
107432024-02-13T21:43:32.178Z
107442024-02-13T21:43:32.178ZVerifying data now
107452024-02-13T21:43:37.028ZNo change after verify
107462024-02-13T21:43:37.028ZLoop: 49 Downstairs dump after verify (and repair):
107472024-02-13T21:43:37.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107482024-02-13T21:43:37.043Z 0 000-019 97 97 97 181 181 181 F F F
107492024-02-13T21:43:37.043Z 1 020-039 95 95 95 176 176 176 F F F
107502024-02-13T21:43:37.043Z 2 040-059 97 97 97 181 181 181 F F F
107512024-02-13T21:43:37.043Z 3 060-079 97 97 97 181 181 181 F F F
107522024-02-13T21:43:37.043Z 4 080-099 98 98 98 182 182 182 F F F
107532024-02-13T21:43:37.043Z 5 100-119 98 98 98 182 182 182 F F F
107542024-02-13T21:43:37.043Z 6 120-139 98 98 98 182 182 182 F F F
107552024-02-13T21:43:37.043Z 7 140-159 98 98 98 182 182 182 F F F
107562024-02-13T21:43:37.043Z 8 160-179 98 98 98 182 182 182 F F F
107572024-02-13T21:43:37.043Z 9 180-199 98 98 98 182 182 182 F F F
107582024-02-13T21:43:37.043Z 10 200-219 98 98 98 182 182 182 F F F
107592024-02-13T21:43:37.043Z 11 220-239 98 98 98 182 182 182 F F F
107602024-02-13T21:43:37.043Z 12 240-259 96 96 96 179 179 179 F F F
107612024-02-13T21:43:37.044Z 13 260-279 96 96 96 177 177 177 F F F
107622024-02-13T21:43:37.047Z 14 280-299 98 98 98 182 182 182 F F F
107632024-02-13T21:43:37.047Z 15 300-319 98 98 98 182 182 182 F F F
107642024-02-13T21:43:37.047Z 16 320-339 98 98 98 182 182 182 F F F
107652024-02-13T21:43:37.047Z 17 340-359 98 98 98 182 182 182 F F F
107662024-02-13T21:43:37.047Z 18 360-379 98 98 98 182 182 182 F F F
107672024-02-13T21:43:37.047Z 19 380-399 92 92 92 170 170 170 F F F
107682024-02-13T21:43:37.047Z 20 400-419 92 92 92 170 170 170 F F F
107692024-02-13T21:43:37.047Z 21 420-439 96 96 96 177 177 177 F F F
107702024-02-13T21:43:37.047Z 22 440-459 90 90 90 164 164 164 F F F
107712024-02-13T21:43:37.047Z 23 460-479 96 96 96 179 179 179 F F F
107722024-02-13T21:43:37.047Z 24 480-499 98 98 98 182 182 182 F F F
107732024-02-13T21:43:37.047Z 25 500-519 98 98 98 182 182 182 F F F
107742024-02-13T21:43:37.047Z 26 520-539 98 98 98 182 182 182 F F F
107752024-02-13T21:43:37.047Z 27 540-559 98 98 98 182 182 182 F F F
107762024-02-13T21:43:37.047Z 28 560-579 91 91 91 169 169 169 F F F
107772024-02-13T21:43:37.047Z 29 580-599 98 98 98 182 182 182 F F F
107782024-02-13T21:43:37.047ZMax gen: 98, Max flush: 182
107792024-02-13T21:43:37.047Z
107802024-02-13T21:43:37.047ZBegin loop 50
107812024-02-13T21:43:37.048ZDownstairs to restart: 2
107822024-02-13T21:43:37.065Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.821934125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
107832024-02-13T21:43:37.065Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822219972Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
107842024-02-13T21:43:37.070Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822275581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
107852024-02-13T21:43:37.070Z{"msg":"Crucible ce936c1d-6ed0-4e04-8258-958d5704a129 has session id: 75ed0544-22f0-422f-b263-638fceb12826","v":0,"name":"crucible","level":30Crucible runtime is spawned
107862024-02-13T21:43:37.070Z,"time":"2024-02-13T21:43:35.822310439Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107872024-02-13T21:43:37.070Z{"msg":"Upstairs opts: Upstairs UUID: ce936c1d-6ed0-4e04-8258-958d5704a129, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822350844Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107882024-02-13T21:43:37.070Z{"msg":"Crucible stats registered with UUID: ce936c1d-6ed0-4e04-8258-958d5704a129","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822387671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107892024-02-13T21:43:37.071Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.82241872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
107902024-02-13T21:43:37.071Z{"msg":"ce936c1d-6ed0-4e04-8258-958d5704a129 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822446639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107912024-02-13T21:43:37.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822460364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107922024-02-13T21:43:37.071Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822474569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107932024-02-13T21:43:37.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822489024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107942024-02-13T21:43:37.071Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822511686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107952024-02-13T21:43:37.071Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822534907Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107962024-02-13T21:43:37.071Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822561368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107972024-02-13T21:43:37.071Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822599924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107982024-02-13T21:43:37.071Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822633882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
107992024-02-13T21:43:37.071Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822673037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108002024-02-13T21:43:37.071Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822793573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108012024-02-13T21:43:37.071Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822849223Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108022024-02-13T21:43:37.071Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.822899695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"io task","client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108032024-02-13T21:43:37.071Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.823589125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108042024-02-13T21:43:37.071Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.823646664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108052024-02-13T21:43:37.071Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.823698595Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108062024-02-13T21:43:37.071Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.823753825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108072024-02-13T21:43:37.071Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824206491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108082024-02-13T21:43:37.071Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824340052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108092024-02-13T21:43:37.071Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824363654Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108102024-02-13T21:43:37.071Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824386676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108112024-02-13T21:43:37.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824625159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108122024-02-13T21:43:37.072Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824680579Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108132024-02-13T21:43:37.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824716306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108142024-02-13T21:43:37.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824902998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108152024-02-13T21:43:37.072Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824941174Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108162024-02-13T21:43:37.072Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.824976502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108172024-02-13T21:43:37.072Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825006071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108182024-02-13T21:43:37.072Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825378406Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108192024-02-13T21:43:37.072Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825419781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108202024-02-13T21:43:37.072Z{"msg":"[0]R flush_numbers[0..12]: [181, 176, 181, 181, 182, 182, 182, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825437095Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108212024-02-13T21:43:37.072Z{"msg":"[0]R generation[0..12]: [97, 95, 97, 97, 98, 98, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825458677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108222024-02-13T21:43:37.072Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825473571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108232024-02-13T21:43:37.072Z{"msg":"[1]R flush_numbers[0..12]: [181, 176, 181, 181, 182, 182, 182, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825487936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108242024-02-13T21:43:37.072Z{"msg":"[1]R generation[0..12]: [97, 95, 97, 97, 98, 98, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825502971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108252024-02-13T21:43:37.072Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825525033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108262024-02-13T21:43:37.072Z{"msg":"[2]R flush_numbers[0..12]: [181, 176, 181, 181, 182, 182, 182, 182, 182, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825539617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108272024-02-13T21:43:37.072Z{"msg":"[2]R generation[0..12]: [97, 95, 97, 97, 98, 98, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.82555975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108282024-02-13T21:43:37.072Z{"msgWait for a query_work_queue command to finish before sending IO
108292024-02-13T21:43:37.072Z":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825580083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108302024-02-13T21:43:37.076Z{"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.82561543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108312024-02-13T21:43:37.077Z{"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825653256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108322024-02-13T21:43:37.077Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825686744Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108332024-02-13T21:43:37.077Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825708466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"":"downstairs","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108342024-02-13T21:43:37.077Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825739095Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108352024-02-13T21:43:37.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825773543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"0","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108362024-02-13T21:43:37.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.82580808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"1","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108372024-02-13T21:43:37.077Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825833841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"client":"2","":"downstairs","session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108382024-02-13T21:43:37.077Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825854943Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108392024-02-13T21:43:37.077Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825880704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108402024-02-13T21:43:37.077Z{"msg":"ce936c1d-6ed0-4e04-8258-958d5704a129 is now active with session: 75ed0544-22f0-422f-b263-638fceb12826","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825894719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108412024-02-13T21:43:37.077Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.825906714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538,"session_id":"75ed0544-22f0-422f-b263-638fceb12826"}
108422024-02-13T21:43:37.077Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:35.82591931Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
108432024-02-13T21:43:37.077ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108442024-02-13T21:43:37.077ZLoading write count information from file "/tmp/test_repair_verify.data"
108452024-02-13T21:43:37.077ZRun Repair workload
108462024-02-13T21:43:37.077Z01/30 Write block 549 len 28672 data: 9 9 11 9 9 9 8
108472024-02-13T21:43:37.077Z02/30 Write block 354 len 32768 data: 9 10 12 11 10 10 9 7
108482024-02-13T21:43:37.077Z03/30 Write block 89 len 20480 data: 5 5 6 7 7
108492024-02-13T21:43:37.077Z04/30 Read block 586 len 28672
108502024-02-13T21:43:37.077Z05/30 Write block 471 len 32768 data: 10 10 11 12 11 8 7 9
108512024-02-13T21:43:37.077Z06/30 Write block 266 len 32768 data: 7 7 8 8 8 8 9 9
108522024-02-13T21:43:37.077Z07/30 Read block 398 len 20480
108532024-02-13T21:43:37.081Z08/30 Read block 156 len 16384
108542024-02-13T21:43:37.081Z09/30 Flush
108552024-02-13T21:43:37.081Z10/30 Write block 220 len 16384 data: 10 10 11 12
108562024-02-13T21:43:37.081Z11/30 Read block 133 len 12288
108572024-02-13T21:43:37.084Z12/30 Write block 480 len 8192 data: 8 9
108582024-02-13T21:43:37.084Z13/30 Read block 86 len 28672
108592024-02-13T21:43:37.084Z14/30 Write block 144 len 16384 data: 11 11 8 8
108602024-02-13T21:43:37.084Z15/30 Read block 210 len 28672
108612024-02-13T21:43:37.090Z16/30 Read block 93 len 36864
108622024-02-13T21:43:37.090Z17/30 Write block 265 len 32768 data: 7 8 8 9 9 9 9 10
108632024-02-13T21:43:37.090Z18/30 Flush
108642024-02-13T21:43:37.093Z19/30 Write block 166 len 16384 data: 5 3 2 3
108652024-02-13T21:43:37.093Z20/30 Write block 557 len 28672 data: 6 7 5 6 6 6 6
108662024-02-13T21:43:37.093Z21/30 Write block 219 len 28672 data: 12 11 11 12 13 10 10
108672024-02-13T21:43:37.093Z22/30 Write block 73 len 24576 data: 7 9 9 9 9 6
108682024-02-13T21:43:37.093Z23/30 Write block 288 len 24576 data: 7 8 8 9 10 12
108692024-02-13T21:43:37.096Z24/30 Read block 32 len 28672
108702024-02-13T21:43:37.096Z25/30 Read block 354 len 16384
108712024-02-13T21:43:37.096Z26/30 Write block 138 len 4096 data: 10
108722024-02-13T21:43:37.096Z27/30 Write block 455 len 36864 data: 7 7 7 8 10 10 8 8 7
108732024-02-13T21:43:37.100Z28/30 Read block 509 len 20480
108742024-02-13T21:43:37.100Z29/30 Write block 35 len 8192 data: 12 12
108752024-02-13T21:43:37.100Z30/30 Read block 274 len 40960
108762024-02-13T21:43:37.103Z----------------------------------------------------------------
108772024-02-13T21:43:37.103Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108782024-02-13T21:43:37.103ZStates: Active Active Active
108792024-02-13T21:43:37.103ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108802024-02-13T21:43:37.103Z 1 Acked 1000 Write 7 Done Done Done false
108812024-02-13T21:43:37.104Z 2 Acked 1001 Write 8 Done Done Sent false
108822024-02-13T21:43:37.104Z 3 Acked 1002 Write 5 Done Done Sent false
108832024-02-13T21:43:37.104Z 4 Acked 1003 Read 7 Done Done Sent false
108842024-02-13T21:43:37.104Z 5 Acked 1004 Write 8 Done Done Sent false
108852024-02-13T21:43:37.104Z 6 Acked 1005 Write 8 Done Done Sent false
108862024-02-13T21:43:37.104Z 7 Acked 1006 Read 5 Done Done Sent false
108872024-02-13T21:43:37.104Z 8 Acked 1007 Read 4 Done Done Sent false
108882024-02-13T21:43:37.104Z 9 Acked 1008 Flush 0 Done Done Sent false
108892024-02-13T21:43:37.104Z 10 Acked 1009 Write 4 Done Done Sent false
108902024-02-13T21:43:37.104Z 11 Acked 1010 Read 3 Done Done Sent false
108912024-02-13T21:43:37.104Z 12 Acked 1011 Write 2 Done Done Sent false
108922024-02-13T21:43:37.104Z 13 Acked 1012 Read 7 Done Done Sent false
108932024-02-13T21:43:37.104Z 14 Acked 1013 Write 4 Done Done Sent false
108942024-02-13T21:43:37.104Z 15 Acked 1014 Read 7 Done Done Sent false
108952024-02-13T21:43:37.104Z 16 Acked 1015 Read 9 Done Done Sent false
108962024-02-13T21:43:37.104Z 17 Acked 1016 Write 8 Done Done Sent false
108972024-02-13T21:43:37.104Z 18 Acked 1017 Flush 0 Done Done Sent false
108982024-02-13T21:43:37.104Z 19 Acked 1018 Write 4 Done Done Sent false
108992024-02-13T21:43:37.104Z 20 Acked 1019 Write 7 Done Done Sent false
109002024-02-13T21:43:37.104Z 21 Acked 1020 Write 7 Done Done Sent false
109012024-02-13T21:43:37.104Z 22 Acked 1021 Write 6 Done Done Sent false
109022024-02-13T21:43:37.104Z 23 Acked 1022 Write 6 Done Done Sent false
109032024-02-13T21:43:37.104Z 24 Acked 1023 Read 7 Done Done Sent false
109042024-02-13T21:43:37.104Z 25 Acked 1024 Read 4 Done Done Sent false
109052024-02-13T21:43:37.105Z 26 Acked 1025 Write 1 Done Done Sent false
109062024-02-13T21:43:37.105Z 27 Acked 1026 Write 9 Done Done Sent false
109072024-02-13T21:43:37.105Z 28 Acked 1027 Read 5 Done Done Sent false
109082024-02-13T21:43:37.105Z 29 Acked 1028 Write 2 Done Done Sent false
109092024-02-13T21:43:37.105Z 30 Acked 1029 Read 10 Done Done Sent false
109102024-02-13T21:43:37.105Z STATES DS:0 DS:1 DS:2 TOTAL
109112024-02-13T21:43:37.105Z New 0 0 0 0
109122024-02-13T21:43:37.105Z Sent 0 0 29 29
109132024-02-13T21:43:37.105Z Done 30 30 1 61
109142024-02-13T21:43:37.105Z Skipped 0 0 0 0
109152024-02-13T21:43:37.105Z Error 0 0 0 0
109162024-02-13T21:43:37.105ZLast Flush: 1017 1017 0
109172024-02-13T21:43:37.105ZDownstairs last five completed:
109182024-02-13T21:43:37.105ZUpstairs last five completed: 30 29 28 27 26
109192024-02-13T21:43:37.105Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:35.857306005Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1538}
109202024-02-13T21:43:37.105ZWrote out file "/tmp/test_repair_verify.data"
109212024-02-13T21:43:37.108Z
109222024-02-13T21:43:37.109ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109232024-02-13T21:43:37.125ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109242024-02-13T21:43:37.125Z 0 000-019 97 97 97 181 181 181 F F F
109252024-02-13T21:43:37.125Z 1 020-039 95 95 95 176 176 176 T T F <---
109262024-02-13T21:43:37.125Z 2 040-059 97 97 97 181 181 181 F F F
109272024-02-13T21:43:37.125Z 3 060-079 97 97 97 181 181 181 T T F <---
109282024-02-13T21:43:37.125Z 4 080-099 100 100 98 183 183 182 F F F <---
109292024-02-13T21:43:37.126Z 5 100-119 98 98 98 182 182 182 F F F
109302024-02-13T21:43:37.126Z 6 120-139 98 98 98 182 182 182 T T F <---
109312024-02-13T21:43:37.126Z 7 140-159 100 100 98 184 184 182 F F F <---
109322024-02-13T21:43:37.126Z 8 160-179 98 98 98 182 182 182 T T F <---
109332024-02-13T21:43:37.126Z 9 180-199 98 98 98 182 182 182 F F F
109342024-02-13T21:43:37.126Z 10 200-219 98 98 98 182 182 182 T T F <---
109352024-02-13T21:43:37.126Z 11 220-239 100 100 98 184 184 182 T T F <---
109362024-02-13T21:43:37.126Z 12 240-259 96 96 96 179 179 179 F F F
109372024-02-13T21:43:37.126Z 13 260-279 100 100 96 184 184 177 F F F <---
109382024-02-13T21:43:37.126Z 14 280-299 98 98 98 182 182 182 T T F <---
109392024-02-13T21:43:37.126Z 15 300-319 98 98 98 182 182 182 F F F
109402024-02-13T21:43:37.129Z 16 320-339 98 98 98 182 182 182 F F F
109412024-02-13T21:43:37.129Z 17 340-359 100 100 98 183 183 182 F F F <---
109422024-02-13T21:43:37.129Z 18 360-379 100 100 98 183 183 182 F F F <---
109432024-02-13T21:43:37.129Z 19 380-399 92 92 92 170 170 170 F F F
109442024-02-13T21:43:37.129Z 20 400-419 92 92 92 170 170 170 F F F
109452024-02-13T21:43:37.129Z 21 420-439 96 96 96 177 177 177 F F F
109462024-02-13T21:43:37.129Z 22 440-459 90 90 90 164 164 164 T T F <---
109472024-02-13T21:43:37.129Z 23 460-479 100 100 96 183 183 179 T T F <---
109482024-02-13T21:43:37.129Z 24 480-499 100 100 98 184 184 182 F F F <---
109492024-02-13T21:43:37.129Z 25 500-519 98 98 98 182 182 182 F F F
109502024-02-13T21:43:37.130Z 26 520-539 98 98 98 182 182 182 F F F
109512024-02-13T21:43:37.130Z 27 540-559 100 100 98 183 183 182 T T T <---
109522024-02-13T21:43:37.130Z 28 560-579 91 91 91 169 169 169 T T F <---
109532024-02-13T21:43:37.130Z 29 580-599 98 98 98 182 182 182 F F F
109542024-02-13T21:43:37.130ZMax gen: 100, Max flush: 184
109552024-02-13T21:43:37.130ZError: Difference in extent metadata found!
109562024-02-13T21:43:37.130ZA Difference in extent metadata is expected here
109572024-02-13T21:43:37.130ZOn loop 50
109582024-02-13T21:43:37.130Z
109592024-02-13T21:43:37.133ZVerifying data now
109602024-02-13T21:43:41.927ZNo change after verify
109612024-02-13T21:43:41.927ZLoop: 50 Downstairs dump after verify (and repair):
109622024-02-13T21:43:41.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109632024-02-13T21:43:41.939Z 0 000-019 97 97 97 181 181 181 F F F
109642024-02-13T21:43:41.939Z 1 020-039 101 101 101 185 185 185 F F F
109652024-02-13T21:43:41.939Z 2 040-059 97 97 97 181 181 181 F F F
109662024-02-13T21:43:41.939Z 3 060-079 101 101 101 185 185 185 F F F
109672024-02-13T21:43:41.939Z 4 080-099 100 100 100 183 183 183 F F F
109682024-02-13T21:43:41.939Z 5 100-119 98 98 98 182 182 182 F F F
109692024-02-13T21:43:41.939Z 6 120-139 101 101 101 185 185 185 F F F
109702024-02-13T21:43:41.939Z 7 140-159 100 100 100 184 184 184 F F F
109712024-02-13T21:43:41.940Z 8 160-179 101 101 101 185 185 185 F F F
109722024-02-13T21:43:41.940Z 9 180-199 98 98 98 182 182 182 F F F
109732024-02-13T21:43:41.940Z 10 200-219 101 101 101 185 185 185 F F F
109742024-02-13T21:43:41.940Z 11 220-239 101 101 101 185 185 185 F F F
109752024-02-13T21:43:41.940Z 12 240-259 96 96 96 179 179 179 F F F
109762024-02-13T21:43:41.940Z 13 260-279 100 100 100 184 184 184 F F F
109772024-02-13T21:43:41.940Z 14 280-299 101 101 101 185 185 185 F F F
109782024-02-13T21:43:41.940Z 15 300-319 98 98 98 182 182 182 F F F
109792024-02-13T21:43:41.940Z 16 320-339 98 98 98 182 182 182 F F F
109802024-02-13T21:43:41.940Z 17 340-359 100 100 100 183 183 183 F F F
109812024-02-13T21:43:41.940Z 18 360-379 100 100 100 183 183 183 F F F
109822024-02-13T21:43:41.941Z 19 380-399 92 92 92 170 170 170 F F F
109832024-02-13T21:43:41.941Z 20 400-419 92 92 92 170 170 170 F F F
109842024-02-13T21:43:41.941Z 21 420-439 96 96 96 177 177 177 F F F
109852024-02-13T21:43:41.941Z 22 440-459 101 101 101 185 185 185 F F F
109862024-02-13T21:43:41.941Z 23 460-479 101 101 101 185 185 185 F F F
109872024-02-13T21:43:41.941Z 24 480-499 100 100 100 184 184 184 F F F
109882024-02-13T21:43:41.941Z 25 500-519 98 98 98 182 182 182 F F F
109892024-02-13T21:43:41.941Z 26 520-539 98 98 98 182 182 182 F F F
109902024-02-13T21:43:41.941Z 27 540-559 101 101 101 185 185 185 F F F
109912024-02-13T21:43:41.942Z 28 560-579 101 101 101 185 185 185 F F F
109922024-02-13T21:43:41.942Z 29 580-599 98 98 98 182 182 182 F F F
109932024-02-13T21:43:41.942ZMax gen: 101, Max flush: 185
109942024-02-13T21:43:41.942Z
109952024-02-13T21:43:41.942ZBegin loop 51
109962024-02-13T21:43:41.942ZDownstairs to restart: 2
109972024-02-13T21:43:41.965Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.709803138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
109982024-02-13T21:43:41.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710036753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
109992024-02-13T21:43:42.005Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710057965Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
110002024-02-13T21:43:42.005Z{"msg":"Crucible b032bbd7-62d4-48b2-a4b7-e928cf4f7473 has session id: df1e8f64-ea7f-4ea9-87b4-b063d952080d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710094042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110012024-02-13T21:43:42.005Z{"msg":"Upstairs opts: Upstairs UUID: b032bbd7-62d4-48b2-a4b7-e928cf4f7473, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.71012817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110022024-02-13T21:43:42.005Z{"msg":"Crucible stats registered with UUID: b032bbd7-62d4-48b2-a4b7-e928cf4f7473","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710168655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110032024-02-13T21:43:42.005ZCrucible runtime is spawned
110042024-02-13T21:43:42.006Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710307445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
110052024-02-13T21:43:42.006Z{"msg":"b032bbd7-62d4-48b2-a4b7-e928cf4f7473 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710390884Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110062024-02-13T21:43:42.006Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710424652Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110072024-02-13T21:43:42.006Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710441306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110082024-02-13T21:43:42.006Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.7104584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110092024-02-13T21:43:42.006Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710492858Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110102024-02-13T21:43:42.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710519278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110112024-02-13T21:43:42.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710557094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110122024-02-13T21:43:42.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710584504Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110132024-02-13T21:43:42.007Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710618292Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110142024-02-13T21:43:42.008Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710644742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110152024-02-13T21:43:42.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710684008Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110162024-02-13T21:43:42.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710711038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110172024-02-13T21:43:42.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.710810862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"io task","client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110182024-02-13T21:43:42.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.711443822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110192024-02-13T21:43:42.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.711485158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110202024-02-13T21:43:42.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.711511678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110212024-02-13T21:43:42.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.711575365Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110222024-02-13T21:43:42.008Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712087989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110232024-02-13T21:43:42.009Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712115809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110242024-02-13T21:43:42.009Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712632991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110252024-02-13T21:43:42.009Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712680144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110262024-02-13T21:43:42.009Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712722589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110272024-02-13T21:43:42.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712762264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110282024-02-13T21:43:42.010Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.71280318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110292024-02-13T21:43:42.010Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.7128285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110302024-02-13T21:43:42.010Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712841715Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110312024-02-13T21:43:42.010Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.712855301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110322024-02-13T21:43:42.010Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713273619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110332024-02-13T21:43:42.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.71371227Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110342024-02-13T21:43:42.011Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713752795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110352024-02-13T21:43:42.011Z{"msg":"[0]R flush_numbers[0..12]: [181, 185, 181, 185, 183, 182, 185, 184, 185, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713777376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110362024-02-13T21:43:42.011Z{"msg":"[0]R generation[0..12]: [97, 101, 97, 101, 100, 98, 101, 100, 101, 98, 101, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713792111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110372024-02-13T21:43:42.011Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713812893Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110382024-02-13T21:43:42.012Z{"msg":"[1]R flush_numbers[0..12]: [181, 185, 181, 185, 183, 182, 185, 184, 185, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713827518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110392024-02-13T21:43:42.012Z{"msg":"[1]R generation[0..12]: [97, 101, 97, 101, 100, 98, 101, 100, 101, 98, 101, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.71384779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110402024-02-13T21:43:42.012Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713862295Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110412024-02-13T21:43:42.012Z{"msg":"[2]R flush_numbers[0..12]: [181, 185, 181, 185, 183, 182, 185, 184, 185, 182, 185, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713894094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
110422024-02-13T21:43:42.012Z,"pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110432024-02-13T21:43:42.013Z{"msg":"[2]R generation[0..12]: [97, 101, 97, 101, 100, 98, 101, 100, 101, 98, 101, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713933139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110442024-02-13T21:43:42.013Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713965848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110452024-02-13T21:43:42.013Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.713981142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110462024-02-13T21:43:42.013Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714006503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110472024-02-13T21:43:42.013Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714040161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110482024-02-13T21:43:42.014Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714074678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"":"downstairs","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110492024-02-13T21:43:42.014Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714091172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110502024-02-13T21:43:42.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714122941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"0","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110512024-02-13T21:43:42.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714158068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"1","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110522024-02-13T21:43:42.014Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714176891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"client":"2","":"downstairs","session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110532024-02-13T21:43:42.014Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714201492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110542024-02-13T21:43:42.014Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714224244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110552024-02-13T21:43:42.014Z{"msg":"b032bbd7-62d4-48b2-a4b7-e928cf4f7473 is now active with session: df1e8f64-ea7f-4ea9-87b4-b063d952080d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714257832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110562024-02-13T21:43:42.014Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714274656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546,"session_id":"df1e8f64-ea7f-4ea9-87b4-b063d952080d"}
110572024-02-13T21:43:42.014Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:40.714287001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
110582024-02-13T21:43:42.014ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110592024-02-13T21:43:42.014ZLoading write count information from file "/tmp/test_repair_verify.data"
110602024-02-13T21:43:42.014ZRun Repair workload
110612024-02-13T21:43:42.014Z01/30 Write block 379 len 8192 data: 10 10
110622024-02-13T21:43:42.015Z02/30 Read block 215 len 20480
110632024-02-13T21:43:42.015Z03/30 Write block 227 len 36864 data: 9 8 7 8 8 7 6 8 7
110642024-02-13T21:43:42.015Z04/30 Write block 537 len 16384 data: 6 7 9 11
110652024-02-13T21:43:42.015Z05/30 Write block 449 len 20480 data: 10 9 8 6 6
110662024-02-13T21:43:42.015Z06/30 Read block 126 len 40960
110672024-02-13T21:43:42.015Z07/30 Write block 340 len 32768 data: 10 9 10 11 9 9 8 9
110682024-02-13T21:43:42.015Z08/30 Write block 130 len 28672 data: 6 9 9 10 8 9 8
110692024-02-13T21:43:42.016Z09/30 Read block 228 len 12288
110702024-02-13T21:43:42.016Z10/30 Write block 452 len 4096 data: 7
110712024-02-13T21:43:42.016Z11/30 Read block 365 len 4096
110722024-02-13T21:43:42.016Z12/30 Write block 408 len 28672 data: 5 7 7 10 7 8 9
110732024-02-13T21:43:42.016Z13/30 Read block 86 len 20480
110742024-02-13T21:43:42.016Z14/30 Write block 317 len 40960 data: 10 10 12 11 11 12 13 12 13 13
110752024-02-13T21:43:42.016Z15/30 Flush
110762024-02-13T21:43:42.045Z16/30 Write block 103 len 24576 data: 13 11 10 8 8 9
110772024-02-13T21:43:42.046Z17/30 Read block 120 len 36864
110782024-02-13T21:43:42.046Z18/30 Read block 9 len 16384
110792024-02-13T21:43:42.046Z19/30 Flush
110802024-02-13T21:43:42.047Z20/30 Write block 457 len 40960 data: 8 9 11 11 9 9 8 10 10 9
110812024-02-13T21:43:42.047Z21/30 Write block 328 len 24576 data: 10 8 7 11 12 11
110822024-02-13T21:43:42.047Z22/30 Write block 158 len 20480 data: 9 8 7 8 8
110832024-02-13T21:43:42.047Z23/30 Flush
110842024-02-13T21:43:42.047Z24/30 Write block 33 len 16384 data: 9 10 13 13
110852024-02-13T21:43:42.047Z25/30 Write block 268 len 32768 data: 10 10 10 10 11 10 10 11
110862024-02-13T21:43:42.048Z26/30 Read block 325 len 40960
110872024-02-13T21:43:42.048Z27/30 Read block 68 len 8192
110882024-02-13T21:43:42.048Z28/30 Write block 296 len 12288 data: 9 9 9
110892024-02-13T21:43:42.048Z29/30 Write block 460 len 16384 data: 12 10 10 9
110902024-02-13T21:43:42.048Z30/30 Read block 440 len 36864
110912024-02-13T21:43:42.048Z----------------------------------------------------------------
110922024-02-13T21:43:42.048Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110932024-02-13T21:43:42.048ZStates: Active Active Active
110942024-02-13T21:43:42.049ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110952024-02-13T21:43:42.049Z 1 Acked 1000 Write 2 Done Done Done false
110962024-02-13T21:43:42.049Z 2 Acked 1001 Read 5 Done Done Sent false
110972024-02-13T21:43:42.049Z 3 Acked 1002 Write 9 Done Done Sent false
110982024-02-13T21:43:42.049Z 4 Acked 1003 Write 4 Done Done Sent false
110992024-02-13T21:43:42.049Z 5 Acked 1004 Write 5 Done Done Sent false
111002024-02-13T21:43:42.049Z 6 Acked 1005 Read 10 Done Done Sent false
111012024-02-13T21:43:42.050Z 7 Acked 1006 Write 8 Done Done Sent false
111022024-02-13T21:43:42.050Z 8 Acked 1007 Write 7 Done Done Sent false
111032024-02-13T21:43:42.050Z 9 Acked 1008 Read 3 Done Done Sent false
111042024-02-13T21:43:42.050Z 10 Acked 1009 Write 1 Done Done Sent false
111052024-02-13T21:43:42.051Z 11 Acked 1010 Read 1 Done Done Sent false
111062024-02-13T21:43:42.051Z 12 Acked 1011 Write 7 Done Done Sent false
111072024-02-13T21:43:42.051Z 13 Acked 1012 Read 5 Done Done Sent false
111082024-02-13T21:43:42.051Z 14 Acked 1013 Write 10 Done Done Sent false
111092024-02-13T21:43:42.051Z 15 Acked 1014 Flush 0 Done Done Sent false
111102024-02-13T21:43:42.051Z 16 Acked 1015 Write 6 Done Done Sent false
111112024-02-13T21:43:42.051Z 17 Acked 1016 Read 9 Done Done Sent false
111122024-02-13T21:43:42.051Z 18 Acked 1017 Read 4 Done Done Sent false
111132024-02-13T21:43:42.051Z 19 Acked 1018 Flush 0 Done Done Sent false
111142024-02-13T21:43:42.052Z 20 Acked 1019 Write 10 Done Done Sent false
111152024-02-13T21:43:42.052Z 21 Acked 1020 Write 6 Done Done Sent false
111162024-02-13T21:43:42.052Z 22 Acked 1021 Write 5 Done Done Sent false
111172024-02-13T21:43:42.052Z 23 Acked 1022 Flush 0 Done Done Sent false
111182024-02-13T21:43:42.052Z 24 Acked 1023 Write 4 Done Done Sent false
111192024-02-13T21:43:42.052Z 25 Acked 1024 Write 8 Done Done Sent false
111202024-02-13T21:43:42.052Z 26 Acked 1025 Read 10 Done Done Sent false
111212024-02-13T21:43:42.053Z 27 Acked 1026 Read 2 Done Done Sent false
111222024-02-13T21:43:42.053Z 28 Acked 1027 Write 3 Done Done Sent false
111232024-02-13T21:43:42.053Z 29 Acked 1028 Write 4 Done Done Sent false
111242024-02-13T21:43:42.053Z 30 Acked 1029 Read 9 Done Sent Sent false
111252024-02-13T21:43:42.053Z STATES DS:0 DS:1 DS:2 TOTAL
111262024-02-13T21:43:42.053Z New 0 0 0 0
111272024-02-13T21:43:42.054Z Sent 0 1 29 30
111282024-02-13T21:43:42.054Z Done 30 29 1 60
111292024-02-13T21:43:42.054Z Skipped 0 0 0 0
111302024-02-13T21:43:42.054Z Error 0 0 0 0
111312024-02-13T21:43:42.054ZLast Flush: 1022 1022 0
111322024-02-13T21:43:42.054ZDownstairs last five completed:
111332024-02-13T21:43:42.054ZUpstairs last five completed: 30 29 28 27 26
111342024-02-13T21:43:42.054Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:40.748046436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1546}
111352024-02-13T21:43:42.055ZWrote out file "/tmp/test_repair_verify.data"
111362024-02-13T21:43:42.055Z
111372024-02-13T21:43:42.055ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111382024-02-13T21:43:42.055ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111392024-02-13T21:43:42.055Z 0 000-019 97 97 97 181 181 181 F F F
111402024-02-13T21:43:42.055Z 1 020-039 101 101 101 185 185 185 T T F <---
111412024-02-13T21:43:42.055Z 2 040-059 97 97 97 181 181 181 F F F
111422024-02-13T21:43:42.056Z 3 060-079 101 101 101 185 185 185 F F F
111432024-02-13T21:43:42.056Z 4 080-099 100 100 100 183 183 183 F F F
111442024-02-13T21:43:42.056Z 5 100-119 102 102 98 187 187 182 F F F <---
111452024-02-13T21:43:42.056Z 6 120-139 102 102 101 186 186 185 F F F <---
111462024-02-13T21:43:42.056Z 7 140-159 102 102 100 188 188 184 F F F <---
111472024-02-13T21:43:42.056Z 8 160-179 102 102 101 188 188 185 F F F <---
111482024-02-13T21:43:42.056Z 9 180-199 98 98 98 182 182 182 F F F
111492024-02-13T21:43:42.057Z 10 200-219 101 101 101 185 185 185 F F F
111502024-02-13T21:43:42.057Z 11 220-239 102 102 101 186 186 185 F F F <---
111512024-02-13T21:43:42.057Z 12 240-259 96 96 96 179 179 179 F F F
111522024-02-13T21:43:42.057Z 13 260-279 100 100 100 184 184 184 T T F <---
111532024-02-13T21:43:42.057Z 14 280-299 101 101 101 185 185 185 T T F <---
111542024-02-13T21:43:42.057Z 15 300-319 102 102 98 186 186 182 F F F <---
111552024-02-13T21:43:42.057Z 16 320-339 102 102 98 188 188 182 F F F <---
111562024-02-13T21:43:42.057Z 17 340-359 102 102 100 186 186 183 F F F <---
111572024-02-13T21:43:42.058Z 18 360-379 102 102 100 186 186 183 F F T <---
111582024-02-13T21:43:42.058Z 19 380-399 102 102 92 186 186 170 F F T <---
111592024-02-13T21:43:42.058Z 20 400-419 102 102 92 186 186 170 F F F <---
111602024-02-13T21:43:42.058Z 21 420-439 96 96 96 177 177 177 F F F
111612024-02-13T21:43:42.058Z 22 440-459 102 102 101 188 188 185 F F F <---
111622024-02-13T21:43:42.058Z 23 460-479 102 102 101 188 188 185 T T F <---
111632024-02-13T21:43:42.059Z 24 480-499 100 100 100 184 184 184 F F F
111642024-02-13T21:43:42.059Z 25 500-519 98 98 98 182 182 182 F F F
111652024-02-13T21:43:42.059Z 26 520-539 102 102 98 186 186 182 F F F <---
111662024-02-13T21:43:42.059Z 27 540-559 102 102 101 186 186 185 F F F <---
111672024-02-13T21:43:42.059Z 28 560-579 101 101 101 185 185 185 F F F
111682024-02-13T21:43:42.059Z 29 580-599 98 98 98 182 182 182 F F F
111692024-02-13T21:43:42.059ZMax gen: 102, Max flush: 188
111702024-02-13T21:43:42.060ZError: Difference in extent metadata found!
111712024-02-13T21:43:42.060ZA Difference in extent metadata is expected here
111722024-02-13T21:43:42.060ZOn loop 51
111732024-02-13T21:43:42.060Z
111742024-02-13T21:43:42.060ZVerifying data now
111752024-02-13T21:43:47.867ZNo change after verify
111762024-02-13T21:43:47.867ZLoop: 51 Downstairs dump after verify (and repair):
111772024-02-13T21:43:47.882ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111782024-02-13T21:43:47.882Z 0 000-019 97 97 97 181 181 181 F F F
111792024-02-13T21:43:47.882Z 1 020-039 103 103 103 189 189 189 F F F
111802024-02-13T21:43:47.882Z 2 040-059 97 97 97 181 181 181 F F F
111812024-02-13T21:43:47.882Z 3 060-079 101 101 101 185 185 185 F F F
111822024-02-13T21:43:47.882Z 4 080-099 100 100 100 183 183 183 F F F
111832024-02-13T21:43:47.882Z 5 100-119 102 102 102 187 187 187 F F F
111842024-02-13T21:43:47.882Z 6 120-139 102 102 102 186 186 186 F F F
111852024-02-13T21:43:47.886Z 7 140-159 102 102 102 188 188 188 F F F
111862024-02-13T21:43:47.886Z 8 160-179 102 102 102 188 188 188 F F F
111872024-02-13T21:43:47.886Z 9 180-199 98 98 98 182 182 182 F F F
111882024-02-13T21:43:47.886Z 10 200-219 101 101 101 185 185 185 F F F
111892024-02-13T21:43:47.886Z 11 220-239 102 102 102 186 186 186 F F F
111902024-02-13T21:43:47.886Z 12 240-259 96 96 96 179 179 179 F F F
111912024-02-13T21:43:47.886Z 13 260-279 103 103 103 189 189 189 F F F
111922024-02-13T21:43:47.886Z 14 280-299 103 103 103 189 189 189 F F F
111932024-02-13T21:43:47.886Z 15 300-319 102 102 102 186 186 186 F F F
111942024-02-13T21:43:47.886Z 16 320-339 102 102 102 188 188 188 F F F
111952024-02-13T21:43:47.887Z 17 340-359 102 102 102 186 186 186 F F F
111962024-02-13T21:43:47.887Z 18 360-379 102 102 102 186 186 186 F F F
111972024-02-13T21:43:47.887Z 19 380-399 102 102 102 186 186 186 F F F
111982024-02-13T21:43:47.887Z 20 400-419 102 102 102 186 186 186 F F F
111992024-02-13T21:43:47.887Z 21 420-439 96 96 96 177 177 177 F F F
112002024-02-13T21:43:47.887Z 22 440-459 102 102 102 188 188 188 F F F
112012024-02-13T21:43:47.887Z 23 460-479 103 103 103 189 189 189 F F F
112022024-02-13T21:43:47.887Z 24 480-499 100 100 100 184 184 184 F F F
112032024-02-13T21:43:47.887Z 25 500-519 98 98 98 182 182 182 F F F
112042024-02-13T21:43:47.887Z 26 520-539 102 102 102 186 186 186 F F F
112052024-02-13T21:43:47.887Z 27 540-559 102 102 102 186 186 186 F F F
112062024-02-13T21:43:47.887Z 28 560-579 101 101 101 185 185 185 F F F
112072024-02-13T21:43:47.887Z 29 580-599 98 98 98 182 182 182 F F F
112082024-02-13T21:43:47.887ZMax gen: 103, Max flush: 189
112092024-02-13T21:43:47.887Z
112102024-02-13T21:43:47.887ZBegin loop 52
112112024-02-13T21:43:47.887ZDownstairs to restart: 1
112122024-02-13T21:43:47.903Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.656654978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
112132024-02-13T21:43:47.904ZCrucible runtime is spawned
112142024-02-13T21:43:47.909Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.656983289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
112152024-02-13T21:43:47.909Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.65706474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
112162024-02-13T21:43:47.909Z{"msg":"Crucible 78894f12-431c-4a17-96bc-a490bad6b7ca has session id: 1fdd242c-c611-47d5-882d-db76584a831e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657105505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112172024-02-13T21:43:47.909Z{"msg":"Upstairs opts: Upstairs UUID: 78894f12-431c-4a17-96bc-a490bad6b7ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.65714628Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112182024-02-13T21:43:47.909Z{"msg":"Crucible stats registered with UUID: 78894f12-431c-4a17-96bc-a490bad6b7ca","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657165303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112192024-02-13T21:43:47.909Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657190854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
112202024-02-13T21:43:47.909Z{"msg":"78894f12-431c-4a17-96bc-a490bad6b7ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657206478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112212024-02-13T21:43:47.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657245444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112222024-02-13T21:43:47.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657290668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112232024-02-13T21:43:47.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657324905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112242024-02-13T21:43:47.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657354615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112252024-02-13T21:43:47.910Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657388722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112262024-02-13T21:43:47.910Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657413693Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112272024-02-13T21:43:47.910Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657451319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112282024-02-13T21:43:47.910Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657496163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112292024-02-13T21:43:47.910Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.65753108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112302024-02-13T21:43:47.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657557131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112312024-02-13T21:43:47.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657648208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112322024-02-13T21:43:47.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.657700869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"io task","client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112332024-02-13T21:43:47.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.658367297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112342024-02-13T21:43:47.910Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.658425946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112352024-02-13T21:43:47.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.658461213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112362024-02-13T21:43:47.910Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.658486564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112372024-02-13T21:43:47.910Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.658881961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112382024-02-13T21:43:47.910Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659022799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112392024-02-13T21:43:47.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659346952Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112402024-02-13T21:43:47.910Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659424454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112412024-02-13T21:43:47.910Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659468588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112422024-02-13T21:43:47.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659494708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112432024-02-13T21:43:47.910Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659530565Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112442024-02-13T21:43:47.911Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659553857Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112452024-02-13T21:43:47.911Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659585975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112462024-02-13T21:43:47.911Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.659618473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112472024-02-13T21:43:47.911Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.65992947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112482024-02-13T21:43:47.915Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660357395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112492024-02-13T21:43:47.916Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660391033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112502024-02-13T21:43:47.916Z{"msg":"[0]R flush_numbers[0..12]: [181, 189, 181, 185, 183, 187, 186, 188, 188, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660406967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112512024-02-13T21:43:47.916Z{"msg":"[0]R generation[0..12]: [97, 103, 97, 101, 100, 102, 102, 102, 102, 98, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.66042822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112522024-02-13T21:43:47.916Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660443234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112532024-02-13T21:43:47.916Z{"msg":"[1]R flush_numbers[0..12]: [181, 189, 181, 185, 183, 187, 186, 188, 188, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660464376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112542024-02-13T21:43:47.916Z{"msg":"[1]R generation[0..12]: [97, 103, 97, 101, 100, 102, 102, 102, 102, 98, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660478431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112552024-02-13T21:43:47.916Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660493396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112562024-02-13T21:43:47.916Z{"msg":"[2]R flush_numbers[0..12]: [181, 189, 181, 185, 183, 187, 186, 188, 188, 182, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660514258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112572024-02-13T21:43:47.916Z{"msg":"[2]R generation[0..12]: [97, 103, 97, 101, 100, 102, 102, 102, 102, 98, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660528513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112582024-02-13T21:43:47.916Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660542568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112592024-02-13T21:43:47.916Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660562131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112602024-02-13T21:43:47.916ZWait for a query_work_queue command to finish before sending IO
112612024-02-13T21:43:47.916Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660578655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112622024-02-13T21:43:47.916Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660598498Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112632024-02-13T21:43:47.916Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660611973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"":"downstairs","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112642024-02-13T21:43:47.916Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660625848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112652024-02-13T21:43:47.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660639083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"0","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112662024-02-13T21:43:47.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660664764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"1","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112672024-02-13T21:43:47.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660678259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"client":"2","":"downstairs","session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112682024-02-13T21:43:47.916Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660691464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112692024-02-13T21:43:47.916Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660714646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112702024-02-13T21:43:47.916Z{"msg":"78894f12-431c-4a17-96bc-a490bad6b7ca is now active with session: 1fdd242c-c611-47d5-882d-db76584a831e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.66073085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112712024-02-13T21:43:47.917Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660756621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554,"session_id":"1fdd242c-c611-47d5-882d-db76584a831e"}
112722024-02-13T21:43:47.917Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:46.660769496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
112732024-02-13T21:43:47.917ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112742024-02-13T21:43:47.917ZLoading write count information from file "/tmp/test_repair_verify.data"
112752024-02-13T21:43:47.917ZRun Repair workload
112762024-02-13T21:43:47.917Z01/30 Write block 438 len 40960 data: 8 8 7 8 8 7 7 10 10 9
112772024-02-13T21:43:47.917Z02/30 Write block 299 len 24576 data: 8 6 5 5 7 7
112782024-02-13T21:43:47.917Z03/30 Write block 51 len 4096 data: 7
112792024-02-13T21:43:47.917Z04/30 Write block 257 len 40960 data: 9 8 7 5 5 5 6 8 8 9
112802024-02-13T21:43:47.917Z05/30 Read block 41 len 20480
112812024-02-13T21:43:47.917Z06/30 Read block 575 len 40960
112822024-02-13T21:43:47.920Z07/30 Write block 470 len 36864 data: 9 11 11 12 13 12 9 8 10
112832024-02-13T21:43:47.920Z08/30 Read block 58 len 40960
112842024-02-13T21:43:47.921Z09/30 Flush
112852024-02-13T21:43:47.924Z10/30 Write block 54 len 40960 data: 7 8 9 10 10 12 10 9 7 6
112862024-02-13T21:43:47.924Z11/30 Read block 515 len 28672
112872024-02-13T21:43:47.924Z12/30 Write block 359 len 20480 data: 11 10 8 7 9
112882024-02-13T21:43:47.927Z13/30 Flush
112892024-02-13T21:43:47.931Z14/30 Flush
112902024-02-13T21:43:47.931Z15/30 Write block 85 len 8192 data: 8 7
112912024-02-13T21:43:47.931Z16/30 Read block 384 len 8192
112922024-02-13T21:43:47.931Z17/30 Write block 37 len 28672 data: 8 9 8 8 8 6 5
112932024-02-13T21:43:47.931Z18/30 Read block 291 len 16384
112942024-02-13T21:43:47.934Z19/30 Write block 526 len 28672 data: 5 5 6 6 6 5 6
112952024-02-13T21:43:47.934Z20/30 Write block 11 len 20480 data: 13 11 13 14 14
112962024-02-13T21:43:47.934Z21/30 Read block 519 len 24576
112972024-02-13T21:43:47.934Z22/30 Read block 515 len 20480
112982024-02-13T21:43:47.937Z23/30 Write block 324 len 20480 data: 13 14 14 12 11
112992024-02-13T21:43:47.938Z24/30 Write block 195 len 16384 data: 12 14 15 14
113002024-02-13T21:43:47.938Z25/30 Flush
113012024-02-13T21:43:47.941Z26/30 Write block 353 len 24576 data: 10 10 11 13 12 11
113022024-02-13T21:43:47.941Z27/30 Read block 356 len 24576
113032024-02-13T21:43:47.944Z28/30 Write block 393 len 40960 data: 7 7 5 5 4 3 3 4 4 5
113042024-02-13T21:43:47.944Z29/30 Write block 249 len 36864 data: 6 6 8 7 7 8 9 9 10
113052024-02-13T21:43:47.944Z30/30 Write block 335 len 8192 data: 13 12
113062024-02-13T21:43:47.944Z----------------------------------------------------------------
113072024-02-13T21:43:47.944Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
113082024-02-13T21:43:47.944ZStates: Active Active Active
113092024-02-13T21:43:47.944ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113102024-02-13T21:43:47.944Z 1 Acked 1000 Write 10 Done Sent Done false
113112024-02-13T21:43:47.944Z 2 Acked 1001 Write 6 Done Sent Done false
113122024-02-13T21:43:47.944Z 3 Acked 1002 Write 1 Done Sent Done false
113132024-02-13T21:43:47.944Z 4 Acked 1003 Write 10 Done Sent Done false
113142024-02-13T21:43:47.944Z 5 Acked 1004 Read 5 Done Sent Done false
113152024-02-13T21:43:47.945Z 6 Acked 1005 Read 10 Done Sent Done false
113162024-02-13T21:43:47.945Z 7 Acked 1006 Write 9 Done Sent Done false
113172024-02-13T21:43:47.945Z 8 Acked 1007 Read 10 Done Sent Done false
113182024-02-13T21:43:47.945Z 9 Acked 1008 Flush 0 Done Sent Done false
113192024-02-13T21:43:47.945Z 10 Acked 1009 Write 10 Done Sent Done false
113202024-02-13T21:43:47.945Z 11 Acked 1010 Read 7 Done Sent Done false
113212024-02-13T21:43:47.945Z 12 Acked 1011 Write 5 Done Sent Done false
113222024-02-13T21:43:47.945Z 13 Acked 1012 Flush 0 Done Sent Done false
113232024-02-13T21:43:47.945Z 14 Acked 1013 Flush 0 Done Sent Done false
113242024-02-13T21:43:47.945Z 15 Acked 1014 Write 2 Done Sent Done false
113252024-02-13T21:43:47.945Z 16 Acked 1015 Read 2 Done Sent Done false
113262024-02-13T21:43:47.945Z 17 Acked 1016 Write 7 Done Sent Done false
113272024-02-13T21:43:47.945Z 18 Acked 1017 Read 4 Done Sent Done false
113282024-02-13T21:43:47.945Z 19 Acked 1018 Write 7 Done Sent Done false
113292024-02-13T21:43:47.945Z 20 Acked 1019 Write 5 Done Sent Done false
113302024-02-13T21:43:47.945Z 21 Acked 1020 Read 6 Done Sent Done false
113312024-02-13T21:43:47.945Z 22 Acked 1021 Read 5 Done Sent Done false
113322024-02-13T21:43:47.945Z 23 Acked 1022 Write 5 Done Sent Done false
113332024-02-13T21:43:47.945Z 24 Acked 1023 Write 4 Done Sent Done false
113342024-02-13T21:43:47.945Z 25 Acked 1024 Flush 0 Done Sent Done false
113352024-02-13T21:43:47.945Z 26 Acked 1025 Write 6 Done Sent Done false
113362024-02-13T21:43:47.945Z 27 Acked 1026 Read 6 Done Sent Done false
113372024-02-13T21:43:47.945Z 28 Acked 1027 Write 10 Done Sent Done false
113382024-02-13T21:43:47.945Z 29 Acked 1028 Write 9 Sent Sent Sent false
113392024-02-13T21:43:47.946Z 30 Acked 1029 Write 2 Sent Sent Sent false
113402024-02-13T21:43:47.946Z STATES DS:0 DS:1 DS:2 TOTAL
113412024-02-13T21:43:47.946Z New 0 0 0 0
113422024-02-13T21:43:47.946Z Sent 2 30 2 34
113432024-02-13T21:43:47.946Z Done 28 0 28 56
113442024-02-13T21:43:47.946Z Skipped 0 0 0 0
113452024-02-13T21:43:47.946Z Error 0 0 0 0
113462024-02-13T21:43:47.946ZLast Flush: 1024 0 1024
113472024-02-13T21:43:47.946ZDownstairs last five completed:
113482024-02-13T21:43:47.946ZUpstairs last five completed: 30 29 28 27 26
113492024-02-13T21:43:47.946Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:46.696553386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1554}
113502024-02-13T21:43:47.946ZWrote out file "/tmp/test_repair_verify.data"
113512024-02-13T21:43:47.949Z
113522024-02-13T21:43:47.953ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113532024-02-13T21:43:47.967ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113542024-02-13T21:43:47.967Z 0 000-019 104 97 104 193 181 193 F F F <---
113552024-02-13T21:43:47.967Z 1 020-039 104 103 104 193 189 193 F F F <---
113562024-02-13T21:43:47.967Z 2 040-059 104 97 104 193 181 193 F F F <---
113572024-02-13T21:43:47.967Z 3 060-079 104 101 104 191 185 191 F F F <---
113582024-02-13T21:43:47.967Z 4 080-099 104 100 104 193 183 193 F F F <---
113592024-02-13T21:43:47.967Z 5 100-119 102 102 102 187 187 187 F F F
113602024-02-13T21:43:47.967Z 6 120-139 102 102 102 186 186 186 F F F
113612024-02-13T21:43:47.967Z 7 140-159 102 102 102 188 188 188 F F F
113622024-02-13T21:43:47.967Z 8 160-179 102 102 102 188 188 188 F F F
113632024-02-13T21:43:47.974Z 9 180-199 104 98 104 193 182 193 F F F <---
113642024-02-13T21:43:47.974Z 10 200-219 101 101 101 185 185 185 F F F
113652024-02-13T21:43:47.974Z 11 220-239 102 102 102 186 186 186 F F F
113662024-02-13T21:43:47.974Z 12 240-259 104 96 104 190 179 190 T F T <---
113672024-02-13T21:43:47.974Z 13 260-279 104 103 104 190 189 190 F F F <---
113682024-02-13T21:43:47.974Z 14 280-299 104 103 104 190 189 190 F F F <---
113692024-02-13T21:43:47.974Z 15 300-319 104 102 104 190 186 190 F F F <---
113702024-02-13T21:43:47.975Z 16 320-339 104 102 104 193 188 193 T F T <---
113712024-02-13T21:43:47.975Z 17 340-359 104 102 104 191 186 191 T F T <---
113722024-02-13T21:43:47.975Z 18 360-379 104 102 104 191 186 191 F F F <---
113732024-02-13T21:43:47.975Z 19 380-399 102 102 102 186 186 186 T F T <---
113742024-02-13T21:43:47.975Z 20 400-419 102 102 102 186 186 186 T F T <---
113752024-02-13T21:43:47.975Z 21 420-439 104 96 104 190 177 190 F F F <---
113762024-02-13T21:43:47.975Z 22 440-459 104 102 104 190 188 190 F F F <---
113772024-02-13T21:43:47.975Z 23 460-479 104 103 104 190 189 190 F F F <---
113782024-02-13T21:43:47.975Z 24 480-499 100 100 100 184 184 184 F F F
113792024-02-13T21:43:47.975Z 25 500-519 98 98 98 182 182 182 F F F
113802024-02-13T21:43:47.975Z 26 520-539 104 102 104 193 186 193 F F F <---
113812024-02-13T21:43:47.975Z 27 540-559 102 102 102 186 186 186 F F F
113822024-02-13T21:43:47.975Z 28 560-579 101 101 101 185 185 185 F F F
113832024-02-13T21:43:47.975Z 29 580-599 98 98 98 182 182 182 F F F
113842024-02-13T21:43:47.975ZMax gen: 104, Max flush: 193
113852024-02-13T21:43:47.975ZError: Difference in extent metadata found!
113862024-02-13T21:43:47.975ZA Difference in extent metadata is expected here
113872024-02-13T21:43:47.975ZOn loop 52
113882024-02-13T21:43:47.975Z
113892024-02-13T21:43:47.978ZVerifying data now
113902024-02-13T21:43:52.853ZNo change after verify
113912024-02-13T21:43:52.853ZLoop: 52 Downstairs dump after verify (and repair):
113922024-02-13T21:43:52.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113932024-02-13T21:43:52.868Z 0 000-019 104 104 104 193 193 193 F F F
113942024-02-13T21:43:52.868Z 1 020-039 104 104 104 193 193 193 F F F
113952024-02-13T21:43:52.868Z 2 040-059 104 104 104 193 193 193 F F F
113962024-02-13T21:43:52.868Z 3 060-079 104 104 104 191 191 191 F F F
113972024-02-13T21:43:52.868Z 4 080-099 104 104 104 193 193 193 F F F
113982024-02-13T21:43:52.869Z 5 100-119 102 102 102 187 187 187 F F F
113992024-02-13T21:43:52.869Z 6 120-139 102 102 102 186 186 186 F F F
114002024-02-13T21:43:52.869Z 7 140-159 102 102 102 188 188 188 F F F
114012024-02-13T21:43:52.869Z 8 160-179 102 102 102 188 188 188 F F F
114022024-02-13T21:43:52.886Z 9 180-199 104 104 104 193 193 193 F F F
114032024-02-13T21:43:52.886Z 10 200-219 101 101 101 185 185 185 F F F
114042024-02-13T21:43:52.887Z 11 220-239 102 102 102 186 186 186 F F F
114052024-02-13T21:43:52.887Z 12 240-259 105 105 105 194 194 194 F F F
114062024-02-13T21:43:52.887Z 13 260-279 104 104 104 190 190 190 F F F
114072024-02-13T21:43:52.887Z 14 280-299 104 104 104 190 190 190 F F F
114082024-02-13T21:43:52.887Z 15 300-319 104 104 104 190 190 190 F F F
114092024-02-13T21:43:52.887Z 16 320-339 105 105 105 194 194 194 F F F
114102024-02-13T21:43:52.887Z 17 340-359 105 105 105 194 194 194 F F F
114112024-02-13T21:43:52.887Z 18 360-379 104 104 104 191 191 191 F F F
114122024-02-13T21:43:52.888Z 19 380-399 105 105 105 194 194 194 F F F
114132024-02-13T21:43:52.888Z 20 400-419 105 105 105 194 194 194 F F F
114142024-02-13T21:43:52.888Z 21 420-439 104 104 104 190 190 190 F F F
114152024-02-13T21:43:52.888Z 22 440-459 104 104 104 190 190 190 F F F
114162024-02-13T21:43:52.888Z 23 460-479 104 104 104 190 190 190 F F F
114172024-02-13T21:43:52.888Z 24 480-499 100 100 100 184 184 184 F F F
114182024-02-13T21:43:52.888Z 25 500-519 98 98 98 182 182 182 F F F
114192024-02-13T21:43:52.888Z 26 520-539 104 104 104 193 193 193 F F F
114202024-02-13T21:43:52.889Z 27 540-559 102 102 102 186 186 186 F F F
114212024-02-13T21:43:52.889Z 28 560-579 101 101 101 185 185 185 F F F
114222024-02-13T21:43:52.889Z 29 580-599 98 98 98 182 182 182 F F F
114232024-02-13T21:43:52.889ZMax gen: 105, Max flush: 194
114242024-02-13T21:43:52.889Z
114252024-02-13T21:43:52.889ZBegin loop 53
114262024-02-13T21:43:52.890ZDownstairs to restart: 0
114272024-02-13T21:43:52.900Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636245701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
114282024-02-13T21:43:52.900Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636483106Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
114292024-02-13T21:43:52.900Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636523201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
114302024-02-13T21:43:52.900Z{"msg":"Crucible 427290fc-6e08-456f-b5c8-1839e7b91596 has session id: 6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27","v":0,"name":"crucible","level":30Crucible runtime is spawned
114312024-02-13T21:43:52.900Z,"time":"2024-02-13T21:43:51.636547682Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114322024-02-13T21:43:52.900Z{"msg":"Upstairs opts: Upstairs UUID: 427290fc-6e08-456f-b5c8-1839e7b91596, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636602842Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114332024-02-13T21:43:52.900Z{"msg":"Crucible stats registered with UUID: 427290fc-6e08-456f-b5c8-1839e7b91596","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636637319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114342024-02-13T21:43:52.901Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636660941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
114352024-02-13T21:43:52.901Z{"msg":"427290fc-6e08-456f-b5c8-1839e7b91596 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636794882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114362024-02-13T21:43:52.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63682926Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114372024-02-13T21:43:52.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636846104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114382024-02-13T21:43:52.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636867446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114392024-02-13T21:43:52.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636881871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114402024-02-13T21:43:52.901Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636895186Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114412024-02-13T21:43:52.901Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.636930503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114422024-02-13T21:43:52.901Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63696742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114432024-02-13T21:43:52.901Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637001477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114442024-02-13T21:43:52.902Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637028597Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114452024-02-13T21:43:52.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63713266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114462024-02-13T21:43:52.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637201795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114472024-02-13T21:43:52.902Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637265502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"io task","client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114482024-02-13T21:43:52.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637867903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114492024-02-13T21:43:52.902Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637908598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114502024-02-13T21:43:52.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637938807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114512024-02-13T21:43:52.902Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.637966657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114522024-02-13T21:43:52.902Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638410956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114532024-02-13T21:43:52.902Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63859364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114542024-02-13T21:43:52.902Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638736668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114552024-02-13T21:43:52.903Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63878578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114562024-02-13T21:43:52.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638887863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114572024-02-13T21:43:52.903Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638932297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114582024-02-13T21:43:52.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638956828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114592024-02-13T21:43:52.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.638986767Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114602024-02-13T21:43:52.903Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639012148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114612024-02-13T21:43:52.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639059681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114622024-02-13T21:43:52.903Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639303243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114632024-02-13T21:43:52.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639701378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114642024-02-13T21:43:52.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639737315Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114652024-02-13T21:43:52.904Z{"msg":"[0]R flush_numbers[0..12]: [193, 193, 193, 191, 193, 187, 186, 188, 188, 193, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639754089Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114662024-02-13T21:43:52.904Z{"msg":"[0]R generation[0..12]: [104, 104, 104, 104, 104, 102, 102, 102, 102, 104, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639768804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114672024-02-13T21:43:52.904Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639790616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114682024-02-13T21:43:52.904Z{"msg":"[1]R flush_numbers[0..12]: [193, 193, 193, 191, 193, 187, 186, 188, 188, 193, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.6398053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114692024-02-13T21:43:52.904Z{"msg":"[1]R generation[0..12]: [104, 104, 104, 104, 104, 102, 102, 102, 102, 104, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639827213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114702024-02-13T21:43:52.904Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639842007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114712024-02-13T21:43:52.904Z{"msg":"[2]R flush_numbers[0..12]: [193, 193, 193, 191, 193, 187, 186, 188, 188, 193, 185, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639868947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114722024-02-13T21:43:52.904Z{"msg":"[2]R generation[0..12]: [104, 104, 104, 104, 104, 102, 102, 102, 102, 104, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639883352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114732024-02-13T21:43:52.904Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639897987Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114742024-02-13T21:43:52.904Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63991746Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114752024-02-13T21:43:52.904ZWait for a query_work_queue command to finish before sending IO
114762024-02-13T21:43:52.905Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639937792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114772024-02-13T21:43:52.905Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639951327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114782024-02-13T21:43:52.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.63997217Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"":"downstairs","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114792024-02-13T21:43:52.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.639986075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114802024-02-13T21:43:52.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640004868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"0","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114812024-02-13T21:43:52.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640019053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"1","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114822024-02-13T21:43:52.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640032288Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"client":"2","":"downstairs","session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114832024-02-13T21:43:52.905Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640046033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114842024-02-13T21:43:52.905Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640058489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114852024-02-13T21:43:52.905Z{"msg":"427290fc-6e08-456f-b5c8-1839e7b91596 is now active with session: 6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640072364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114862024-02-13T21:43:52.905Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640104272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562,"session_id":"6f2b1dbc-7105-4ec5-8f97-95d4c8b6ab27"}
114872024-02-13T21:43:52.906Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:51.640134401Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
114882024-02-13T21:43:52.906ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114892024-02-13T21:43:52.906ZLoading write count information from file "/tmp/test_repair_verify.data"
114902024-02-13T21:43:52.906ZRun Repair workload
114912024-02-13T21:43:52.906Z01/30 Write block 149 len 32768 data: 6 6 6 5 7 7 8 7
114922024-02-13T21:43:52.906Z02/30 Write block 114 len 40960 data: 7 7 9 10 8 8 7 8 9 9
114932024-02-13T21:43:52.906Z03/30 Write block 416 len 4096 data: 14
114942024-02-13T21:43:52.906Z04/30 Flush
114952024-02-13T21:43:52.911Z05/30 Write block 527 len 8192 data: 6 7
114962024-02-13T21:43:52.911Z06/30 Write block 66 len 16384 data: 4 4 7 9
114972024-02-13T21:43:52.911Z07/30 Read block 193 len 36864
114982024-02-13T21:43:52.911Z08/30 Write block 549 len 12288 data: 10 10 12
114992024-02-13T21:43:52.911Z09/30 Write block 284 len 4096 data: 6
115002024-02-13T21:43:52.911Z10/30 Write block 227 len 40960 data: 10 9 8 9 9 8 7 9 8 6
115012024-02-13T21:43:52.911Z11/30 Read block 175 len 8192
115022024-02-13T21:43:52.911Z12/30 Flush
115032024-02-13T21:43:52.911Z13/30 Write block 379 len 16384 data: 11 11 11 11
115042024-02-13T21:43:52.912Z14/30 Write block 500 len 12288 data: 9 5 7
115052024-02-13T21:43:52.912Z15/30 Read block 352 len 16384
115062024-02-13T21:43:52.912Z16/30 Write block 70 len 32768 data: 9 8 10 8 10 10 10 10
115072024-02-13T21:43:52.912Z17/30 Write block 285 len 20480 data: 7 6 5 8 9
115082024-02-13T21:43:52.912Z18/30 Write block 331 len 28672 data: 12 13 12 10 14 13 10
115092024-02-13T21:43:52.912Z19/30 Read block 334 len 32768
115102024-02-13T21:43:52.912Z20/30 Write block 256 len 8192 data: 10 11
115112024-02-13T21:43:52.917Z21/30 Write block 400 len 40960 data: 5 5 6 6 6 6 4 5 6 8
115122024-02-13T21:43:52.917Z22/30 Flush
115132024-02-13T21:43:52.918Z23/30 Write block 557 len 36864 data: 7 8 6 7 7 7 7 9 10
115142024-02-13T21:43:52.918Z24/30 Read block 425 len 4096
115152024-02-13T21:43:52.918Z25/30 Read block 162 len 24576
115162024-02-13T21:43:52.923Z26/30 Read block 327 len 12288
115172024-02-13T21:43:52.923Z27/30 Write block 20 len 28672 data: 10 10 11 9 9 9 7
115182024-02-13T21:43:52.923Z28/30 Read block 86 len 4096
115192024-02-13T21:43:52.923Z29/30 Write block 223 len 12288 data: 14 11 11
115202024-02-13T21:43:52.923Z30/30 Read block 185 len 4096
115212024-02-13T21:43:52.923Z----------------------------------------------------------------
115222024-02-13T21:43:52.924Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
115232024-02-13T21:43:52.924ZStates: Active Active Active
115242024-02-13T21:43:52.924ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115252024-02-13T21:43:52.924Z 1 Acked 1000 Write 8 Done Done Done false
115262024-02-13T21:43:52.924Z 2 Acked 1001 Write 10 Done Done Done false
115272024-02-13T21:43:52.924Z 3 Acked 1002 Write 1 Sent Done Done false
115282024-02-13T21:43:52.924Z 4 Acked 1003 Flush 0 Sent Done Done false
115292024-02-13T21:43:52.925Z 5 Acked 1004 Write 2 Sent Done Done false
115302024-02-13T21:43:52.925Z 6 Acked 1005 Write 4 Sent Done Done false
115312024-02-13T21:43:52.925Z 7 Acked 1006 Read 9 Sent Done Done false
115322024-02-13T21:43:52.925Z 8 Acked 1007 Write 3 Sent Done Done false
115332024-02-13T21:43:52.925Z 9 Acked 1008 Write 1 Sent Done Done false
115342024-02-13T21:43:52.926Z 10 Acked 1009 Write 10 Sent Done Done false
115352024-02-13T21:43:52.926Z 11 Acked 1010 Read 2 Sent Done Done false
115362024-02-13T21:43:52.926Z 12 Acked 1011 Flush 0 Sent Done Done false
115372024-02-13T21:43:52.926Z 13 Acked 1012 Write 4 Sent Done Done false
115382024-02-13T21:43:52.926Z 14 Acked 1013 Write 3 Sent Done Done false
115392024-02-13T21:43:52.926Z 15 Acked 1014 Read 4 Sent Done Done false
115402024-02-13T21:43:52.927Z 16 Acked 1015 Write 8 Sent Done Done false
115412024-02-13T21:43:52.927Z 17 Acked 1016 Write 5 Sent Done Done false
115422024-02-13T21:43:52.927Z 18 Acked 1017 Write 7 Sent Done Done false
115432024-02-13T21:43:52.927Z 19 Acked 1018 Read 8 Sent Done Done false
115442024-02-13T21:43:52.927Z 20 Acked 1019 Write 2 Sent Done Done false
115452024-02-13T21:43:52.927Z 21 Acked 1020 Write 10 Sent Done Done false
115462024-02-13T21:43:52.927Z 22 Acked 1021 Flush 0 Sent Done Done false
115472024-02-13T21:43:52.927Z 23 Acked 1022 Write 9 Sent Done Done false
115482024-02-13T21:43:52.928Z 24 Acked 1023 Read 1 Sent Done Done false
115492024-02-13T21:43:52.928Z 25 Acked 1024 Read 6 Sent Done Done false
115502024-02-13T21:43:52.928Z 26 Acked 1025 Read 3 Sent Done Done false
115512024-02-13T21:43:52.928Z 27 Acked 1026 Write 7 Sent Done Done false
115522024-02-13T21:43:52.928Z 28 Acked 1027 Read 1 Sent Done Done false
115532024-02-13T21:43:52.928Z 29 Acked 1028 Write 3 Sent Done Done false
115542024-02-13T21:43:52.928Z 30 Acked 1029 Read 1 Sent Done Done false
115552024-02-13T21:43:52.928Z STATES DS:0 DS:1 DS:2 TOTAL
115562024-02-13T21:43:52.928Z New 0 0 0 0
115572024-02-13T21:43:52.928Z Sent 28 0 0 28
115582024-02-13T21:43:52.928Z Done 2 30 30 62
115592024-02-13T21:43:52.929Z Skipped 0 0 0 0
115602024-02-13T21:43:52.929Z Error 0 0 0 0
115612024-02-13T21:43:52.929ZLast Flush: 0 1021 1021
115622024-02-13T21:43:52.929ZDownstairs last five completed:
115632024-02-13T21:43:52.929ZUpstairs last five completed: 30 29 28 27 26
115642024-02-13T21:43:52.929Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:51.672523902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1562}
115652024-02-13T21:43:52.933ZWrote out file "/tmp/test_repair_verify.data"
115662024-02-13T21:43:52.933Z
115672024-02-13T21:43:52.933ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115682024-02-13T21:43:52.947ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115692024-02-13T21:43:52.947Z 0 000-019 104 104 104 193 193 193 F F F
115702024-02-13T21:43:52.947Z 1 020-039 104 104 104 193 193 193 F T T <---
115712024-02-13T21:43:52.948Z 2 040-059 104 104 104 193 193 193 F F F
115722024-02-13T21:43:52.948Z 3 060-079 104 106 106 191 197 197 F F F <---
115732024-02-13T21:43:52.948Z 4 080-099 104 104 104 193 193 193 F F F
115742024-02-13T21:43:52.948Z 5 100-119 102 106 106 187 195 195 T F F <---
115752024-02-13T21:43:52.948Z 6 120-139 102 106 106 186 195 195 T F F <---
115762024-02-13T21:43:52.948Z 7 140-159 102 106 106 188 195 195 T F F <---
115772024-02-13T21:43:52.948Z 8 160-179 102 102 102 188 188 188 F F F
115782024-02-13T21:43:52.948Z 9 180-199 104 104 104 193 193 193 F F F
115792024-02-13T21:43:52.948Z 10 200-219 101 101 101 185 185 185 F F F
115802024-02-13T21:43:52.949Z 11 220-239 102 106 106 186 196 196 F T T <---
115812024-02-13T21:43:52.949Z 12 240-259 105 106 106 194 197 197 F F F <---
115822024-02-13T21:43:52.956Z 13 260-279 104 104 104 190 190 190 F F F
115832024-02-13T21:43:52.956Z 14 280-299 104 106 106 190 197 197 F F F <---
115842024-02-13T21:43:52.956Z 15 300-319 104 104 104 190 190 190 F F F
115852024-02-13T21:43:52.956Z 16 320-339 105 106 106 194 197 197 F F F <---
115862024-02-13T21:43:52.956Z 17 340-359 105 105 105 194 194 194 F F F
115872024-02-13T21:43:52.957Z 18 360-379 104 106 106 191 197 197 F F F <---
115882024-02-13T21:43:52.957Z 19 380-399 105 106 106 194 197 197 F F F <---
115892024-02-13T21:43:52.957Z 20 400-419 105 106 106 194 197 197 F F F <---
115902024-02-13T21:43:52.957Z 21 420-439 104 104 104 190 190 190 F F F
115912024-02-13T21:43:52.957Z 22 440-459 104 104 104 190 190 190 F F F
115922024-02-13T21:43:52.957Z 23 460-479 104 104 104 190 190 190 F F F
115932024-02-13T21:43:52.957Z 24 480-499 100 100 100 184 184 184 F F F
115942024-02-13T21:43:52.957Z 25 500-519 98 106 106 182 197 197 F F F <---
115952024-02-13T21:43:52.958Z 26 520-539 104 106 106 193 196 196 F F F <---
115962024-02-13T21:43:52.958Z 27 540-559 102 106 106 186 196 196 F T T <---
115972024-02-13T21:43:52.958Z 28 560-579 101 101 101 185 185 185 F T T <---
115982024-02-13T21:43:52.958Z 29 580-599 98 98 98 182 182 182 F F F
115992024-02-13T21:43:52.958ZMax gen: 106, Max flush: 197
116002024-02-13T21:43:52.958ZError: Difference in extent metadata found!
116012024-02-13T21:43:52.959ZA Difference in extent metadata is expected here
116022024-02-13T21:43:52.959ZOn loop 53
116032024-02-13T21:43:52.959Z
116042024-02-13T21:43:52.959ZVerifying data now
116052024-02-13T21:43:58.712ZNo change after verify
116062024-02-13T21:43:58.713ZLoop: 53 Downstairs dump after verify (and repair):
116072024-02-13T21:43:58.728ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116082024-02-13T21:43:58.728Z 0 000-019 104 104 104 193 193 193 F F F
116092024-02-13T21:43:58.728Z 1 020-039 107 107 107 198 198 198 F F F
116102024-02-13T21:43:58.728Z 2 040-059 104 104 104 193 193 193 F F F
116112024-02-13T21:43:58.728Z 3 060-079 106 106 106 197 197 197 F F F
116122024-02-13T21:43:58.728Z 4 080-099 104 104 104 193 193 193 F F F
116132024-02-13T21:43:58.728Z 5 100-119 106 106 106 195 195 195 F F F
116142024-02-13T21:43:58.728Z 6 120-139 106 106 106 195 195 195 F F F
116152024-02-13T21:43:58.728Z 7 140-159 106 106 106 195 195 195 F F F
116162024-02-13T21:43:58.728Z 8 160-179 102 102 102 188 188 188 F F F
116172024-02-13T21:43:58.728Z 9 180-199 104 104 104 193 193 193 F F F
116182024-02-13T21:43:58.728Z 10 200-219 101 101 101 185 185 185 F F F
116192024-02-13T21:43:58.728Z 11 220-239 107 107 107 198 198 198 F F F
116202024-02-13T21:43:58.728Z 12 240-259 106 106 106 197 197 197 F F F
116212024-02-13T21:43:58.728Z 13 260-279 104 104 104 190 190 190 F F F
116222024-02-13T21:43:58.728Z 14 280-299 106 106 106 197 197 197 F F F
116232024-02-13T21:43:58.731Z 15 300-319 104 104 104 190 190 190 F F F
116242024-02-13T21:43:58.731Z 16 320-339 106 106 106 197 197 197 F F F
116252024-02-13T21:43:58.731Z 17 340-359 105 105 105 194 194 194 F F F
116262024-02-13T21:43:58.731Z 18 360-379 106 106 106 197 197 197 F F F
116272024-02-13T21:43:58.732Z 19 380-399 106 106 106 197 197 197 F F F
116282024-02-13T21:43:58.732Z 20 400-419 106 106 106 197 197 197 F F F
116292024-02-13T21:43:58.732Z 21 420-439 104 104 104 190 190 190 F F F
116302024-02-13T21:43:58.732Z 22 440-459 104 104 104 190 190 190 F F F
116312024-02-13T21:43:58.732Z 23 460-479 104 104 104 190 190 190 F F F
116322024-02-13T21:43:58.732Z 24 480-499 100 100 100 184 184 184 F F F
116332024-02-13T21:43:58.732Z 25 500-519 106 106 106 197 197 197 F F F
116342024-02-13T21:43:58.732Z 26 520-539 106 106 106 196 196 196 F F F
116352024-02-13T21:43:58.732Z 27 540-559 107 107 107 198 198 198 F F F
116362024-02-13T21:43:58.732Z 28 560-579 107 107 107 198 198 198 F F F
116372024-02-13T21:43:58.732Z 29 580-599 98 98 98 182 182 182 F F F
116382024-02-13T21:43:58.732ZMax gen: 107, Max flush: 198
116392024-02-13T21:43:58.732Z
116402024-02-13T21:43:58.732ZBegin loop 54
116412024-02-13T21:43:58.732ZDownstairs to restart: 2
116422024-02-13T21:43:58.749Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.497918272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
116432024-02-13T21:43:58.754Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498213945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
116442024-02-13T21:43:58.754Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498255749Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
116452024-02-13T21:43:58.754Z{"msg":"Crucible d7ac4852-274f-4e25-b64e-5c0b5c2c6a0e has session id: b877fff0-6f68-4023-85ce-4bbe63f1a4b9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498283869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116462024-02-13T21:43:58.754ZCrucible runtime is spawned
116472024-02-13T21:43:58.754Z{"msg":"Upstairs opts: Upstairs UUID: d7ac4852-274f-4e25-b64e-5c0b5c2c6a0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498340848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116482024-02-13T21:43:58.754Z{"msg":"Crucible stats registered with UUID: d7ac4852-274f-4e25-b64e-5c0b5c2c6a0e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498381404Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116492024-02-13T21:43:58.754Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498409563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
116502024-02-13T21:43:58.754Z{"msg":"d7ac4852-274f-4e25-b64e-5c0b5c2c6a0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498542665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116512024-02-13T21:43:58.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498577722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116522024-02-13T21:43:58.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498595146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116532024-02-13T21:43:58.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498622246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116542024-02-13T21:43:58.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498643289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116552024-02-13T21:43:58.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498678036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116562024-02-13T21:43:58.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498703967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116572024-02-13T21:43:58.755Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498726368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116582024-02-13T21:43:58.755Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498793384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116592024-02-13T21:43:58.755Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498837958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116602024-02-13T21:43:58.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498865738Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116612024-02-13T21:43:58.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498904934Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116622024-02-13T21:43:58.755Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.498932764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"io task","client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116632024-02-13T21:43:58.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.499629831Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116642024-02-13T21:43:58.755Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.499667267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116652024-02-13T21:43:58.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.499683332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116662024-02-13T21:43:58.755Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.499704734Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116672024-02-13T21:43:58.755Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500206032Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116682024-02-13T21:43:58.755Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500329377Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116692024-02-13T21:43:58.755Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500468896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116702024-02-13T21:43:58.755Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500499875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116712024-02-13T21:43:58.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500676291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116722024-02-13T21:43:58.755Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500712638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116732024-02-13T21:43:58.755Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500738339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116742024-02-13T21:43:58.755Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500799587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116752024-02-13T21:43:58.755Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.50084633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116762024-02-13T21:43:58.756Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.500863703Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116772024-02-13T21:43:58.756Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501070758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116782024-02-13T21:43:58.756Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501493805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116792024-02-13T21:43:58.756Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501536939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116802024-02-13T21:43:58.760Z{"msg":"[0]R flush_numbers[0..12]: [193, 198, 193, 197, 193, 195, 195, 195, 188, 193, 185, 198]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501565019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116812024-02-13T21:43:58.760Z{"msg":"[0]R generation[0..12]: [104, 107, 104, 106, 104, 106, 106, 106, 102, 104, 101, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501604655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116822024-02-13T21:43:58.760Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501640052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116832024-02-13T21:43:58.760Z{"msg":"[1]R flush_numbers[0..12]: [193, 198, 193, 197, 193, 195, 195, 195, 188, 193, 185, 198]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501663783Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"Wait for a query_work_queue command to finish before sending IO
116842024-02-13T21:43:58.760Zdownstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116852024-02-13T21:43:58.760Z{"msg":"[1]R generation[0..12]: [104, 107, 104, 106, 104, 106, 106, 106, 102, 104, 101, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.50169965Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116862024-02-13T21:43:58.760Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501737466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116872024-02-13T21:43:58.760Z{"msg":"[2]R flush_numbers[0..12]: [193, 198, 193, 197, 193, 195, 195, 195, 188, 193, 185, 198]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501769995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116882024-02-13T21:43:58.760Z{"msg":"[2]R generation[0..12]: [104, 107, 104, 106, 104, 106, 106, 106, 102, 104, 101, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501794975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116892024-02-13T21:43:58.760Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501814188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116902024-02-13T21:43:58.760Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.50183859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116912024-02-13T21:43:58.760Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501852865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116922024-02-13T21:43:58.760Z{"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.50186604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116932024-02-13T21:43:58.760Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.5018916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"":"downstairs","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116942024-02-13T21:43:58.760Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501914872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116952024-02-13T21:43:58.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501941162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"0","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116962024-02-13T21:43:58.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501965044Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"1","":"downstairs","session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116972024-02-13T21:43:58.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.501992274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"client":"2","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116982024-02-13T21:43:58.761Zb877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
116992024-02-13T21:43:58.761Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.502019294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
117002024-02-13T21:43:58.761Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.502044825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
117012024-02-13T21:43:58.761Z{"msg":"d7ac4852-274f-4e25-b64e-5c0b5c2c6a0e is now active with session: b877fff0-6f68-4023-85ce-4bbe63f1a4b9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.502067716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
117022024-02-13T21:43:58.761Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.502097026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570,"session_id":"b877fff0-6f68-4023-85ce-4bbe63f1a4b9"}
117032024-02-13T21:43:58.761Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:43:57.502119188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
117042024-02-13T21:43:58.761ZLoading write count information from file "/tmp/test_repair_verify.data"
117052024-02-13T21:43:58.761ZRun Repair workload
117062024-02-13T21:43:58.761Z01/30 Write block 576 len 24576 data: 4 5 5 3 3 4
117072024-02-13T21:43:58.761Z02/30 Read block 147 len 40960
117082024-02-13T21:43:58.761Z03/30 Read block 194 len 28672
117092024-02-13T21:43:58.761Z04/30 Flush
117102024-02-13T21:43:58.764Z05/30 Write block 511 len 28672 data: 12 11 11 13 12 11 14
117112024-02-13T21:43:58.764Z06/30 Write block 472 len 32768 data: 12 13 14 13 10 9 11 8
117122024-02-13T21:43:58.764Z07/30 Write block 408 len 40960 data: 7 9 8 11 8 9 10 11 15 15
117132024-02-13T21:43:58.764Z08/30 Read block 529 len 8192
117142024-02-13T21:43:58.765Z09/30 Read block 240 len 28672
117152024-02-13T21:43:58.765Z10/30 Write block 493 len 4096 data: 8
117162024-02-13T21:43:58.768Z11/30 Read block 23 len 8192
117172024-02-13T21:43:58.768Z12/30 Read block 4 len 40960
117182024-02-13T21:43:58.768Z13/30 Write block 539 len 20480 data: 10 12 12 10 10
117192024-02-13T21:43:58.768Z14/30 Write block 384 len 16384 data: 7 7 8 8
117202024-02-13T21:43:58.768Z15/30 Flush
117212024-02-13T21:43:58.771Z16/30 Write block 423 len 16384 data: 10 9 9 10
117222024-02-13T21:43:58.774Z17/30 Write block 433 len 24576 data: 9 11 11 7 7 9
117232024-02-13T21:43:58.774Z18/30 Write block 53 len 20480 data: 6 8 9 10 11
117242024-02-13T21:43:58.774Z19/30 Read block 569 len 12288
117252024-02-13T21:43:58.777Z20/30 Read block 308 len 40960
117262024-02-13T21:43:58.777Z21/30 Write block 178 len 20480 data: 8 7 8 7 6
117272024-02-13T21:43:58.777Z22/30 Flush
117282024-02-13T21:43:58.793Z23/30 Write block 464 len 40960 data: 11 11 10 10 8 9 10 12 13 14
117292024-02-13T21:43:58.793Z24/30 Read block 585 len 12288
117302024-02-13T21:43:58.793Z25/30 Read block 510 len 8192
117312024-02-13T21:43:58.793Z26/30 Write block 409 len 12288 data: 10 9 12
117322024-02-13T21:43:58.793Z27/30 Write block 212 len 36864 data: 7 7 7 8 10 11 12 13 12
117332024-02-13T21:43:58.793Z28/30 Write block 449 len 40960 data: 11 10 9 8 7 6 8 8 9 10
117342024-02-13T21:43:58.793Z29/30 Write block 425 len 40960 data: 10 11 9 10 12 10 11 13 10 12
117352024-02-13T21:43:58.793Z30/30 Write block 143 len 16384 data: 11 12 12 9
117362024-02-13T21:43:58.793Z----------------------------------------------------------------
117372024-02-13T21:43:58.793Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117382024-02-13T21:43:58.793ZStates: Active Active Active
117392024-02-13T21:43:58.793ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117402024-02-13T21:43:58.793Z 1 Acked 1000 Write 6 Done Done Done false
117412024-02-13T21:43:58.794Z 2 Acked 1001 Read 10 Done Done Done false
117422024-02-13T21:43:58.794Z 3 Acked 1002 Read 7 Done Done Done false
117432024-02-13T21:43:58.794Z 4 Acked 1003 Flush 0 Done Done Sent false
117442024-02-13T21:43:58.794Z 5 Acked 1004 Write 7 Done Done Sent false
117452024-02-13T21:43:58.794Z 6 Acked 1005 Write 8 Done Done Sent false
117462024-02-13T21:43:58.794Z 7 Acked 1006 Write 10 Done Done Sent false
117472024-02-13T21:43:58.794Z 8 Acked 1007 Read 2 Done Done Sent false
117482024-02-13T21:43:58.794Z 9 Acked 1008 Read 7 Done Done Sent false
117492024-02-13T21:43:58.794Z 10 Acked 1009 Write 1 Done Done Sent false
117502024-02-13T21:43:58.794Z 11 Acked 1010 Read 2 Done Done Sent false
117512024-02-13T21:43:58.794Z 12 Acked 1011 Read 10 Done Done Sent false
117522024-02-13T21:43:58.794Z 13 Acked 1012 Write 5 Done Done Sent false
117532024-02-13T21:43:58.794Z 14 Acked 1013 Write 4 Done Done Sent false
117542024-02-13T21:43:58.794Z 15 Acked 1014 Flush 0 Done Done Sent false
117552024-02-13T21:43:58.794Z 16 Acked 1015 Write 4 Done Done Sent false
117562024-02-13T21:43:58.794Z 17 Acked 1016 Write 6 Done Done Sent false
117572024-02-13T21:43:58.794Z 18 Acked 1017 Write 5 Done Done Sent false
117582024-02-13T21:43:58.795Z 19 Acked 1018 Read 3 Done Done Sent false
117592024-02-13T21:43:58.795Z 20 Acked 1019 Read 10 Done Done Sent false
117602024-02-13T21:43:58.795Z 21 Acked 1020 Write 5 Done Done Sent false
117612024-02-13T21:43:58.795Z 22 Acked 1021 Flush 0 Done Done Sent false
117622024-02-13T21:43:58.795Z 23 Acked 1022 Write 10 Done Done Sent false
117632024-02-13T21:43:58.795Z 24 Acked 1023 Read 3 Done Done Sent false
117642024-02-13T21:43:58.795Z 25 Acked 1024 Read 2 Done Done Sent false
117652024-02-13T21:43:58.795Z 26 Acked 1025 Write 3 Done Done Sent false
117662024-02-13T21:43:58.795Z 27 Acked 1026 Write 9 Done Done Sent false
117672024-02-13T21:43:58.795Z 28 Acked 1027 Write 10 Done Done Sent false
117682024-02-13T21:43:58.795Z 29 Acked 1028 Write 10 Sent Sent Sent false
117692024-02-13T21:43:58.795Z 30 Acked 1029 Write 4 Sent Sent Sent false
117702024-02-13T21:43:58.795Z STATES DS:0 DS:1 DS:2 TOTAL
117712024-02-13T21:43:58.795Z New 0 0 0 0
117722024-02-13T21:43:58.795Z Sent 2 2 27 31
117732024-02-13T21:43:58.795Z Done 28 28 3 59
117742024-02-13T21:43:58.795Z Skipped 0 0 0 0
117752024-02-13T21:43:58.795Z Error 0 0 0 0
117762024-02-13T21:43:58.795ZLast Flush: 1021 1021 0
117772024-02-13T21:43:58.796ZDownstairs last five completed:
117782024-02-13T21:43:58.796ZUpstairs last five completed: 30 29 28 27 26
117792024-02-13T21:43:58.796Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:43:57.534992744Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1570}
117802024-02-13T21:43:58.796ZWrote out file "/tmp/test_repair_verify.data"
117812024-02-13T21:43:58.796Z
117822024-02-13T21:43:58.801ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117832024-02-13T21:43:58.815ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117842024-02-13T21:43:58.815Z 0 000-019 104 104 104 193 193 193 F F F
117852024-02-13T21:43:58.815Z 1 020-039 107 107 107 198 198 198 F F F
117862024-02-13T21:43:58.815Z 2 040-059 108 108 104 201 201 193 F F F <---
117872024-02-13T21:43:58.815Z 3 060-079 106 106 106 197 197 197 F F F
117882024-02-13T21:43:58.815Z 4 080-099 104 104 104 193 193 193 F F F
117892024-02-13T21:43:58.822Z 5 100-119 106 106 106 195 195 195 F F F
117902024-02-13T21:43:58.822Z 6 120-139 106 106 106 195 195 195 F F F
117912024-02-13T21:43:58.822Z 7 140-159 106 106 106 195 195 195 T T F <---
117922024-02-13T21:43:58.822Z 8 160-179 108 108 102 201 201 188 F F F <---
117932024-02-13T21:43:58.822Z 9 180-199 108 108 104 201 201 193 F F F <---
117942024-02-13T21:43:58.822Z 10 200-219 101 101 101 185 185 185 T T F <---
117952024-02-13T21:43:58.822Z 11 220-239 107 107 107 198 198 198 T T F <---
117962024-02-13T21:43:58.822Z 12 240-259 106 106 106 197 197 197 F F F
117972024-02-13T21:43:58.822Z 13 260-279 104 104 104 190 190 190 F F F
117982024-02-13T21:43:58.822Z 14 280-299 106 106 106 197 197 197 F F F
117992024-02-13T21:43:58.822Z 15 300-319 104 104 104 190 190 190 F F F
118002024-02-13T21:43:58.822Z 16 320-339 106 106 106 197 197 197 F F F
118012024-02-13T21:43:58.822Z 17 340-359 105 105 105 194 194 194 F F F
118022024-02-13T21:43:58.822Z 18 360-379 106 106 106 197 197 197 F F F
118032024-02-13T21:43:58.822Z 19 380-399 108 108 106 200 200 197 F F F <---
118042024-02-13T21:43:58.822Z 20 400-419 108 108 106 200 200 197 T T F <---
118052024-02-13T21:43:58.823Z 21 420-439 108 108 104 201 201 190 T T F <---
118062024-02-13T21:43:58.823Z 22 440-459 104 104 104 190 190 190 T T F <---
118072024-02-13T21:43:58.823Z 23 460-479 108 108 104 200 200 190 T T F <---
118082024-02-13T21:43:58.823Z 24 480-499 108 108 100 200 200 184 F F F <---
118092024-02-13T21:43:58.823Z 25 500-519 108 108 106 200 200 197 F F F <---
118102024-02-13T21:43:58.823Z 26 520-539 108 108 106 200 200 196 F F F <---
118112024-02-13T21:43:58.823Z 27 540-559 108 108 107 200 200 198 F F F <---
118122024-02-13T21:43:58.823Z 28 560-579 108 108 107 199 199 198 F F T <---
118132024-02-13T21:43:58.823Z 29 580-599 108 108 98 199 199 182 F F T <---
118142024-02-13T21:43:58.823ZMax gen: 108, Max flush: 201
118152024-02-13T21:43:58.823ZError: Difference in extent metadata found!
118162024-02-13T21:43:58.823ZA Difference in extent metadata is expected here
118172024-02-13T21:43:58.823ZOn loop 54
118182024-02-13T21:43:58.823Z
118192024-02-13T21:43:58.823ZVerifying data now
118202024-02-13T21:44:04.621ZNo change after verify
118212024-02-13T21:44:04.621ZLoop: 54 Downstairs dump after verify (and repair):
118222024-02-13T21:44:04.635ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118232024-02-13T21:44:04.635Z 0 000-019 104 104 104 193 193 193 F F F
118242024-02-13T21:44:04.635Z 1 020-039 107 107 107 198 198 198 F F F
118252024-02-13T21:44:04.635Z 2 040-059 108 108 108 201 201 201 F F F
118262024-02-13T21:44:04.635Z 3 060-079 106 106 106 197 197 197 F F F
118272024-02-13T21:44:04.635Z 4 080-099 104 104 104 193 193 193 F F F
118282024-02-13T21:44:04.635Z 5 100-119 106 106 106 195 195 195 F F F
118292024-02-13T21:44:04.635Z 6 120-139 106 106 106 195 195 195 F F F
118302024-02-13T21:44:04.635Z 7 140-159 109 109 109 202 202 202 F F F
118312024-02-13T21:44:04.636Z 8 160-179 108 108 108 201 201 201 F F F
118322024-02-13T21:44:04.636Z 9 180-199 108 108 108 201 201 201 F F F
118332024-02-13T21:44:04.639Z 10 200-219 109 109 109 202 202 202 F F F
118342024-02-13T21:44:04.639Z 11 220-239 109 109 109 202 202 202 F F F
118352024-02-13T21:44:04.639Z 12 240-259 106 106 106 197 197 197 F F F
118362024-02-13T21:44:04.640Z 13 260-279 104 104 104 190 190 190 F F F
118372024-02-13T21:44:04.640Z 14 280-299 106 106 106 197 197 197 F F F
118382024-02-13T21:44:04.640Z 15 300-319 104 104 104 190 190 190 F F F
118392024-02-13T21:44:04.640Z 16 320-339 106 106 106 197 197 197 F F F
118402024-02-13T21:44:04.640Z 17 340-359 105 105 105 194 194 194 F F F
118412024-02-13T21:44:04.640Z 18 360-379 106 106 106 197 197 197 F F F
118422024-02-13T21:44:04.640Z 19 380-399 108 108 108 200 200 200 F F F
118432024-02-13T21:44:04.640Z 20 400-419 109 109 109 202 202 202 F F F
118442024-02-13T21:44:04.640Z 21 420-439 109 109 109 202 202 202 F F F
118452024-02-13T21:44:04.640Z 22 440-459 109 109 109 202 202 202 F F F
118462024-02-13T21:44:04.640Z 23 460-479 109 109 109 202 202 202 F F F
118472024-02-13T21:44:04.640Z 24 480-499 108 108 108 200 200 200 F F F
118482024-02-13T21:44:04.640Z 25 500-519 108 108 108 200 200 200 F F F
118492024-02-13T21:44:04.640Z 26 520-539 108 108 108 200 200 200 F F F
118502024-02-13T21:44:04.640Z 27 540-559 108 108 108 200 200 200 F F F
118512024-02-13T21:44:04.640Z 28 560-579 108 108 108 199 199 199 F F F
118522024-02-13T21:44:04.640Z 29 580-599 108 108 108 199 199 199 F F F
118532024-02-13T21:44:04.640ZMax gen: 109, Max flush: 202
118542024-02-13T21:44:04.640Z
118552024-02-13T21:44:04.640ZBegin loop 55
118562024-02-13T21:44:04.640ZDownstairs to restart: 1
118572024-02-13T21:44:04.659Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403150971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
118582024-02-13T21:44:04.660Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403408278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
118592024-02-13T21:44:04.660Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40342841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
118602024-02-13T21:44:04.676Z{"msg":"Crucible b0ea5929-bc69-4e40-a890-44d3bb1a8392 has session id: e81f40f9-a51a-4b90-b875-83e8d6e1fef6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403459699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118612024-02-13T21:44:04.677ZCrucible runtime is spawned
118622024-02-13T21:44:04.677Z{"msg":"Upstairs opts: Upstairs UUID: b0ea5929-bc69-4e40-a890-44d3bb1a8392, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403524016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118632024-02-13T21:44:04.677Z{"msg":"Crucible stats registered with UUID: b0ea5929-bc69-4e40-a890-44d3bb1a8392","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403563441Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118642024-02-13T21:44:04.677Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403588822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
118652024-02-13T21:44:04.678Z{"msg":"b0ea5929-bc69-4e40-a890-44d3bb1a8392 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403717295Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118662024-02-13T21:44:04.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40376075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118672024-02-13T21:44:04.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403796877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118682024-02-13T21:44:04.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403839191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118692024-02-13T21:44:04.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403877227Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118702024-02-13T21:44:04.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403911725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118712024-02-13T21:44:04.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40395289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118722024-02-13T21:44:04.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.403987297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118732024-02-13T21:44:04.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404042037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118742024-02-13T21:44:04.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40408961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118752024-02-13T21:44:04.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404147709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118762024-02-13T21:44:04.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404169691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118772024-02-13T21:44:04.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404212766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"io task","client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118782024-02-13T21:44:04.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404854463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118792024-02-13T21:44:04.680Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404895638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118802024-02-13T21:44:04.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404942491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118812024-02-13T21:44:04.680Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.404992453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118822024-02-13T21:44:04.680Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.405481166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118832024-02-13T21:44:04.680Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40552465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118842024-02-13T21:44:04.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.405763123Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118852024-02-13T21:44:04.680Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.405797891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118862024-02-13T21:44:04.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406062735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118872024-02-13T21:44:04.681Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406104549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118882024-02-13T21:44:04.681Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40613055Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118892024-02-13T21:44:04.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406164068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118902024-02-13T21:44:04.682Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406190029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118912024-02-13T21:44:04.682Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406216999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118922024-02-13T21:44:04.682Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406383558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118932024-02-13T21:44:04.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406842722Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118942024-02-13T21:44:04.683Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40687411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118952024-02-13T21:44:04.683Z{"msg":"[0]R flush_numbers[0..12]: [193, 198, 201, 197, 193, 195, 195, 202, 201, 201, 202, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406900001Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118962024-02-13T21:44:04.683Z{"msg":"[0]R generation[0..12]: [104, 107, 108, 106, 104, 106, 106, 109, 108, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406933759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118972024-02-13T21:44:04.683Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.406972825Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118982024-02-13T21:44:04.683Z{"msg":"[1]R flush_numbers[0..12]: [193, 198, 201, 197, 193, 195, 195, 202, 201, 201, 202, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407002454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
118992024-02-13T21:44:04.683Z{"msg":"[1]R generation[0..12]: [104, 107, 108, 106, 104, 106, 106, 109, 108, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407020247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119002024-02-13T21:44:04.683Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407044019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119012024-02-13T21:44:04.684Z{"msg":"[2]R flush_numbers[0..12]: [193, 198, 201, 197, 193, 195, 195, 202, 201, 201, 202, 202]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407083444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119022024-02-13T21:44:04.684ZWait for a query_work_queue command to finish before sending IO
119032024-02-13T21:44:04.684Z{"msg":"[2]R generation[0..12]: [104, 107, 108, 106, 104, 106, 106, 109, 108, 108, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407118721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119042024-02-13T21:44:04.684Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407155148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119052024-02-13T21:44:04.685Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.4071778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119062024-02-13T21:44:04.685Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407192285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119072024-02-13T21:44:04.685Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407207309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119082024-02-13T21:44:04.685Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407238158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"":"downstairs","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119092024-02-13T21:44:04.685Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407272106Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119102024-02-13T21:44:04.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407298027Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"0","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119112024-02-13T21:44:04.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407312351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"1","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119122024-02-13T21:44:04.685Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407334493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"client":"2","":"downstairs","session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119132024-02-13T21:44:04.686Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.40737107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119142024-02-13T21:44:04.686Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407399879Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119152024-02-13T21:44:04.686Z{"msg":"b0ea5929-bc69-4e40-a890-44d3bb1a8392 is now active with session: e81f40f9-a51a-4b90-b875-83e8d6e1fef6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407414854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119162024-02-13T21:44:04.686Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407445393Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578,"session_id":"e81f40f9-a51a-4b90-b875-83e8d6e1fef6"}
119172024-02-13T21:44:04.686Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:03.407477841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
119182024-02-13T21:44:04.686ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119192024-02-13T21:44:04.686ZLoading write count information from file "/tmp/test_repair_verify.data"
119202024-02-13T21:44:04.686ZRun Repair workload
119212024-02-13T21:44:04.686Z01/30 Write block 503 len 8192 data: 6 8
119222024-02-13T21:44:04.686Z02/30 Write block 340 len 16384 data: 11 10 11 12
119232024-02-13T21:44:04.687Z03/30 Read block 275 len 16384
119242024-02-13T21:44:04.720Z04/30 Read block 397 len 32768
119252024-02-13T21:44:04.721Z05/30 Write block 245 len 36864 data: 5 5 5 5 7 7 9 8 8
119262024-02-13T21:44:04.721Z06/30 Write block 67 len 40960 data: 5 8 10 10 9 11 9 11 11 11
119272024-02-13T21:44:04.721Z07/30 Write block 218 len 20480 data: 13 14 13 12 13
119282024-02-13T21:44:04.721Z08/30 Write block 454 len 28672 data: 7 9 9 10 11 12 13
119292024-02-13T21:44:04.721Z09/30 Read block 208 len 40960
119302024-02-13T21:44:04.721Z10/30 Flush
119312024-02-13T21:44:04.721Z11/30 Write block 256 len 4096 data: 11
119322024-02-13T21:44:04.721Z12/30 Write block 342 len 36864 data: 12 13 10 10 9 10 9 7 7
119332024-02-13T21:44:04.722Z13/30 Read block 542 len 20480
119342024-02-13T21:44:04.732Z14/30 Read block 8 len 40960
119352024-02-13T21:44:04.732Z15/30 Read block 533 len 8192
119362024-02-13T21:44:04.732Z16/30 Write block 587 len 20480 data: 8 7 8 9 8
119372024-02-13T21:44:04.732Z17/30 Write block 102 len 24576 data: 15 14 12 11 9 9
119382024-02-13T21:44:04.732Z18/30 Read block 69 len 12288
119392024-02-13T21:44:04.732Z19/30 Write block 193 len 8192 data: 10 12
119402024-02-13T21:44:04.733Z20/30 Write block 242 len 40960 data: 9 8 7 6 6 6 6 8 8 10
119412024-02-13T21:44:04.733Z21/30 Read block 49 len 8192
119422024-02-13T21:44:04.733Z22/30 Read block 140 len 36864
119432024-02-13T21:44:04.733Z23/30 Read block 162 len 32768
119442024-02-13T21:44:04.733Z24/30 Read block 481 len 40960
119452024-02-13T21:44:04.733Z25/30 Read block 506 len 32768
119462024-02-13T21:44:04.733Z26/30 Flush
119472024-02-13T21:44:04.733Z27/30 Write block 341 len 16384 data: 11 13 14 11
119482024-02-13T21:44:04.734Z28/30 Write block 360 len 24576 data: 11 9 8 10 8 7
119492024-02-13T21:44:04.734Z29/30 Read block 129 len 20480
119502024-02-13T21:44:04.734Z30/30 Write block 266 len 12288 data: 10 9 11
119512024-02-13T21:44:04.734Z----------------------------------------------------------------
119522024-02-13T21:44:04.734Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119532024-02-13T21:44:04.734ZStates: Active Active Active
119542024-02-13T21:44:04.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119552024-02-13T21:44:04.735Z 1 Acked 1000 Write 2 Done Done Done false
119562024-02-13T21:44:04.735Z 2 Acked 1001 Write 4 Done Done Done false
119572024-02-13T21:44:04.735Z 3 Acked 1002 Read 4 Done Done Done false
119582024-02-13T21:44:04.735Z 4 Acked 1003 Read 8 Done Sent Done false
119592024-02-13T21:44:04.735Z 5 Acked 1004 Write 9 Done Sent Done false
119602024-02-13T21:44:04.735Z 6 Acked 1005 Write 10 Done Sent Done false
119612024-02-13T21:44:04.736Z 7 Acked 1006 Write 5 Done Sent Done false
119622024-02-13T21:44:04.736Z 8 Acked 1007 Write 7 Done Sent Done false
119632024-02-13T21:44:04.736Z 9 Acked 1008 Read 10 Done Sent Done false
119642024-02-13T21:44:04.736Z 10 Acked 1009 Flush 0 Done Sent Done false
119652024-02-13T21:44:04.736Z 11 Acked 1010 Write 1 Done Sent Done false
119662024-02-13T21:44:04.736Z 12 Acked 1011 Write 9 Done Sent Done false
119672024-02-13T21:44:04.736Z 13 Acked 1012 Read 5 Done Sent Done false
119682024-02-13T21:44:04.736Z 14 Acked 1013 Read 10 Done Sent Done false
119692024-02-13T21:44:04.736Z 15 Acked 1014 Read 2 Done Sent Done false
119702024-02-13T21:44:04.736Z 16 Acked 1015 Write 5 Done Sent Done false
119712024-02-13T21:44:04.737Z 17 Acked 1016 Write 6 Done Sent Done false
119722024-02-13T21:44:04.737Z 18 Acked 1017 Read 3 Done Sent Done false
119732024-02-13T21:44:04.737Z 19 Acked 1018 Write 2 Done Sent Done false
119742024-02-13T21:44:04.737Z 20 Acked 1019 Write 10 Done Sent Done false
119752024-02-13T21:44:04.737Z 21 Acked 1020 Read 2 Done Sent Done false
119762024-02-13T21:44:04.737Z 22 Acked 1021 Read 9 Done Sent Done false
119772024-02-13T21:44:04.737Z 23 Acked 1022 Read 8 Done Sent Done false
119782024-02-13T21:44:04.738Z 24 Acked 1023 Read 10 Done Sent Done false
119792024-02-13T21:44:04.738Z 25 Acked 1024 Read 8 Done Sent Done false
119802024-02-13T21:44:04.738Z 26 Acked 1025 Flush 0 Done Sent Done false
119812024-02-13T21:44:04.738Z 27 Acked 1026 Write 4 Done Sent Done false
119822024-02-13T21:44:04.738Z 28 Acked 1027 Write 6 Done Sent Done false
119832024-02-13T21:44:04.738Z 29 Acked 1028 Read 5 Done Sent Done false
119842024-02-13T21:44:04.738Z 30 Acked 1029 Write 3 Sent Sent Sent false
119852024-02-13T21:44:04.738Z STATES DS:0 DS:1 DS:2 TOTAL
119862024-02-13T21:44:04.739Z New 0 0 0 0
119872024-02-13T21:44:04.739Z Sent 1 27 1 29
119882024-02-13T21:44:04.739Z Done 29 3 29 61
119892024-02-13T21:44:04.739Z Skipped 0 0 0 0
119902024-02-13T21:44:04.739Z Error 0 0 0 0
119912024-02-13T21:44:04.739ZLast Flush: 1025 0 1025
119922024-02-13T21:44:04.740ZDownstairs last five completed:
119932024-02-13T21:44:04.740ZUpstairs last five completed: 30 29 28 27 26
119942024-02-13T21:44:04.740Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:03.439810593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1578}
119952024-02-13T21:44:04.740ZWrote out file "/tmp/test_repair_verify.data"
119962024-02-13T21:44:04.740Z
119972024-02-13T21:44:04.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119982024-02-13T21:44:04.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119992024-02-13T21:44:04.766Z 0 000-019 104 104 104 193 193 193 F F F
120002024-02-13T21:44:04.766Z 1 020-039 107 107 107 198 198 198 F F F
120012024-02-13T21:44:04.767Z 2 040-059 108 108 108 201 201 201 F F F
120022024-02-13T21:44:04.767Z 3 060-079 110 106 110 203 197 203 F F F <---
120032024-02-13T21:44:04.767Z 4 080-099 104 104 104 193 193 193 F F F
120042024-02-13T21:44:04.767Z 5 100-119 110 106 110 204 195 204 F F F <---
120052024-02-13T21:44:04.767Z 6 120-139 106 106 106 195 195 195 F F F
120062024-02-13T21:44:04.768Z 7 140-159 109 109 109 202 202 202 F F F
120072024-02-13T21:44:04.768Z 8 160-179 108 108 108 201 201 201 F F F
120082024-02-13T21:44:04.768Z 9 180-199 110 108 110 204 201 204 F F F <---
120092024-02-13T21:44:04.768Z 10 200-219 110 109 110 203 202 203 F F F <---
120102024-02-13T21:44:04.768Z 11 220-239 110 109 110 203 202 203 F F F <---
120112024-02-13T21:44:04.768Z 12 240-259 110 106 110 204 197 204 F F F <---
120122024-02-13T21:44:04.768Z 13 260-279 104 104 104 190 190 190 T F T <---
120132024-02-13T21:44:04.768Z 14 280-299 106 106 106 197 197 197 F F F
120142024-02-13T21:44:04.768Z 15 300-319 104 104 104 190 190 190 F F F
120152024-02-13T21:44:04.769Z 16 320-339 106 106 106 197 197 197 F F F
120162024-02-13T21:44:04.769Z 17 340-359 110 105 110 204 194 204 T T T <---
120172024-02-13T21:44:04.769Z 18 360-379 106 106 106 197 197 197 T F T <---
120182024-02-13T21:44:04.769Z 19 380-399 108 108 108 200 200 200 F F F
120192024-02-13T21:44:04.769Z 20 400-419 109 109 109 202 202 202 F F F
120202024-02-13T21:44:04.769Z 21 420-439 109 109 109 202 202 202 F F F
120212024-02-13T21:44:04.769Z 22 440-459 110 109 110 203 202 203 F F F <---
120222024-02-13T21:44:04.769Z 23 460-479 110 109 110 203 202 203 F F F <---
120232024-02-13T21:44:04.770Z 24 480-499 108 108 108 200 200 200 F F F
120242024-02-13T21:44:04.770Z 25 500-519 110 108 110 203 200 203 F T F <---
120252024-02-13T21:44:04.770Z 26 520-539 108 108 108 200 200 200 F F F
120262024-02-13T21:44:04.770Z 27 540-559 108 108 108 200 200 200 F F F
120272024-02-13T21:44:04.770Z 28 560-579 108 108 108 199 199 199 F F F
120282024-02-13T21:44:04.770Z 29 580-599 110 108 110 204 199 204 F F F <---
120292024-02-13T21:44:04.770ZMax gen: 110, Max flush: 204
120302024-02-13T21:44:04.770ZError: Difference in extent metadata found!
120312024-02-13T21:44:04.770ZA Difference in extent metadata is expected here
120322024-02-13T21:44:04.770ZOn loop 55
120332024-02-13T21:44:04.770Z
120342024-02-13T21:44:04.771ZVerifying data now
120352024-02-13T21:44:10.357ZNo change after verify
120362024-02-13T21:44:10.357ZLoop: 55 Downstairs dump after verify (and repair):
120372024-02-13T21:44:10.372ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120382024-02-13T21:44:10.372Z 0 000-019 104 104 104 193 193 193 F F F
120392024-02-13T21:44:10.372Z 1 020-039 107 107 107 198 198 198 F F F
120402024-02-13T21:44:10.372Z 2 040-059 108 108 108 201 201 201 F F F
120412024-02-13T21:44:10.372Z 3 060-079 110 110 110 203 203 203 F F F
120422024-02-13T21:44:10.372Z 4 080-099 104 104 104 193 193 193 F F F
120432024-02-13T21:44:10.373Z 5 100-119 110 110 110 204 204 204 F F F
120442024-02-13T21:44:10.373Z 6 120-139 106 106 106 195 195 195 F F F
120452024-02-13T21:44:10.373Z 7 140-159 109 109 109 202 202 202 F F F
120462024-02-13T21:44:10.378Z 8 160-179 108 108 108 201 201 201 F F F
120472024-02-13T21:44:10.378Z 9 180-199 110 110 110 204 204 204 F F F
120482024-02-13T21:44:10.378Z 10 200-219 110 110 110 203 203 203 F F F
120492024-02-13T21:44:10.378Z 11 220-239 110 110 110 203 203 203 F F F
120502024-02-13T21:44:10.378Z 12 240-259 110 110 110 204 204 204 F F F
120512024-02-13T21:44:10.378Z 13 260-279 111 111 111 205 205 205 F F F
120522024-02-13T21:44:10.378Z 14 280-299 106 106 106 197 197 197 F F F
120532024-02-13T21:44:10.378Z 15 300-319 104 104 104 190 190 190 F F F
120542024-02-13T21:44:10.378Z 16 320-339 106 106 106 197 197 197 F F F
120552024-02-13T21:44:10.378Z 17 340-359 111 111 111 205 205 205 F F F
120562024-02-13T21:44:10.378Z 18 360-379 111 111 111 205 205 205 F F F
120572024-02-13T21:44:10.379Z 19 380-399 108 108 108 200 200 200 F F F
120582024-02-13T21:44:10.379Z 20 400-419 109 109 109 202 202 202 F F F
120592024-02-13T21:44:10.379Z 21 420-439 109 109 109 202 202 202 F F F
120602024-02-13T21:44:10.379Z 22 440-459 110 110 110 203 203 203 F F F
120612024-02-13T21:44:10.379Z 23 460-479 110 110 110 203 203 203 F F F
120622024-02-13T21:44:10.379Z 24 480-499 108 108 108 200 200 200 F F F
120632024-02-13T21:44:10.379Z 25 500-519 110 110 110 203 203 203 F F F
120642024-02-13T21:44:10.379Z 26 520-539 108 108 108 200 200 200 F F F
120652024-02-13T21:44:10.379Z 27 540-559 108 108 108 200 200 200 F F F
120662024-02-13T21:44:10.379Z 28 560-579 108 108 108 199 199 199 F F F
120672024-02-13T21:44:10.379Z 29 580-599 110 110 110 204 204 204 F F F
120682024-02-13T21:44:10.379ZMax gen: 111, Max flush: 205
120692024-02-13T21:44:10.379Z
120702024-02-13T21:44:10.379ZBegin loop 56
120712024-02-13T21:44:10.379ZDownstairs to restart: 2
120722024-02-13T21:44:10.395Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139316017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
120732024-02-13T21:44:10.395Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139567266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
120742024-02-13T21:44:10.401Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139586549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
120752024-02-13T21:44:10.401ZCrucible runtime is spawned
120762024-02-13T21:44:10.401Z{"msg":"Crucible 40258812-aa28-4884-b927-d7010171a831 has session id: 72d77dae-bd84-47dc-8dee-f995dfaa2e19","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139635541Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120772024-02-13T21:44:10.401Z{"msg":"Upstairs opts: Upstairs UUID: 40258812-aa28-4884-b927-d7010171a831, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139674337Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120782024-02-13T21:44:10.401Z{"msg":"Crucible stats registered with UUID: 40258812-aa28-4884-b927-d7010171a831","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139701527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120792024-02-13T21:44:10.401Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139740523Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
120802024-02-13T21:44:10.401Z{"msg":"40258812-aa28-4884-b927-d7010171a831 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139761275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120812024-02-13T21:44:10.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139774241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120822024-02-13T21:44:10.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139789015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120832024-02-13T21:44:10.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139811437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120842024-02-13T21:44:10.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139826062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120852024-02-13T21:44:10.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139856171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120862024-02-13T21:44:10.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139890309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120872024-02-13T21:44:10.402Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139915389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120882024-02-13T21:44:10.402Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139946688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120892024-02-13T21:44:10.402Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.139974218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120902024-02-13T21:44:10.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140151524Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120912024-02-13T21:44:10.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140209563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120922024-02-13T21:44:10.402Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140257695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"io task","client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120932024-02-13T21:44:10.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140898683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120942024-02-13T21:44:10.402Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140932611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120952024-02-13T21:44:10.402Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140948305Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120962024-02-13T21:44:10.402Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.140963089Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120972024-02-13T21:44:10.402Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.141579026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120982024-02-13T21:44:10.402Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.141636805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
120992024-02-13T21:44:10.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142113012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121002024-02-13T21:44:10.402Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142162154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121012024-02-13T21:44:10.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142182407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121022024-02-13T21:44:10.403Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142196302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121032024-02-13T21:44:10.403Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142218324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121042024-02-13T21:44:10.403Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142232149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121052024-02-13T21:44:10.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142245164Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121062024-02-13T21:44:10.403Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142259359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121072024-02-13T21:44:10.403Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.142715144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121082024-02-13T21:44:10.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143129723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121092024-02-13T21:44:10.408Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.14316731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121102024-02-13T21:44:10.408Z{"msg":"[0]R flush_numbers[0..12]: [193, 198, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143183454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121112024-02-13T21:44:10.408Z{"msg":"[0]R generation[0..12]: [104, 107, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143205866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121122024-02-13T21:44:10.408Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.14322073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121132024-02-13T21:44:10.408Z{"msg":"[1]R flush_numbers[0..12]: [193, 198, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143241883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121142024-02-13T21:44:10.408Z{"msg":"[1]R generation[0..12]: [104, 107, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143256197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121152024-02-13T21:44:10.408Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.1432766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121162024-02-13T21:44:10.408Z{"msg":"[2]R flush_numbers[0..12]: [193, 198, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
121172024-02-13T21:44:10.408Z,"time":"2024-02-13T21:44:09.143291165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121182024-02-13T21:44:10.408Z{"msg":"[2]R generation[0..12]: [104, 107, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143307629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121192024-02-13T21:44:10.408Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143323123Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121202024-02-13T21:44:10.408Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143344605Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121212024-02-13T21:44:10.408Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.14335883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121222024-02-13T21:44:10.408Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143378463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121232024-02-13T21:44:10.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143391928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"":"downstairs","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121242024-02-13T21:44:10.409Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143405323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121252024-02-13T21:44:10.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143424287Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"0","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121262024-02-13T21:44:10.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143438562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"1","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121272024-02-13T21:44:10.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143458574Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"client":"2","":"downstairs","session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121282024-02-13T21:44:10.409Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143472059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121292024-02-13T21:44:10.409Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143490943Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121302024-02-13T21:44:10.409Z{"msg":"40258812-aa28-4884-b927-d7010171a831 is now active with session: 72d77dae-bd84-47dc-8dee-f995dfaa2e19","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143522951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121312024-02-13T21:44:10.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143543093Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586,"session_id":"72d77dae-bd84-47dc-8dee-f995dfaa2e19"}
121322024-02-13T21:44:10.409Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:09.143556019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
121332024-02-13T21:44:10.409ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121342024-02-13T21:44:10.409ZLoading write count information from file "/tmp/test_repair_verify.data"
121352024-02-13T21:44:10.409ZRun Repair workload
121362024-02-13T21:44:10.409Z01/30 Flush
121372024-02-13T21:44:10.409Z02/30 Write block 316 len 20480 data: 10 11 11 13 12
121382024-02-13T21:44:10.409Z03/30 Read block 183 len 4096
121392024-02-13T21:44:10.409Z04/30 Write block 388 len 32768 data: 8 7 7 6 6 8 8 6
121402024-02-13T21:44:10.409Z05/30 Read block 174 len 16384
121412024-02-13T21:44:10.409Z06/30 Write block 394 len 36864 data: 9 7 6 5 4 4 6 6 7
121422024-02-13T21:44:10.409Z07/30 Write block 29 len 24576 data: 7 6 9 10 10 11
121432024-02-13T21:44:10.413Z08/30 Write block 380 len 40960 data: 12 12 12 9 8 8 9 9 9 8
121442024-02-13T21:44:10.413Z09/30 Write block 551 len 28672 data: 13 10 10 10 9 9 8
121452024-02-13T21:44:10.413Z10/30 Read block 320 len 28672
121462024-02-13T21:44:10.413Z11/30 Read block 315 len 24576
121472024-02-13T21:44:10.413Z12/30 Write block 383 len 16384 data: 10 9 9 10
121482024-02-13T21:44:10.413Z13/30 Read block 116 len 4096
121492024-02-13T21:44:10.413Z14/30 Flush
121502024-02-13T21:44:10.416Z15/30 Write block 476 len 4096 data: 11
121512024-02-13T21:44:10.416Z16/30 Write block 378 len 24576 data: 10 12 13 13 13 11
121522024-02-13T21:44:10.416Z17/30 Write block 255 len 20480 data: 10 12 12 9 8
121532024-02-13T21:44:10.419Z18/30 Write block 483 len 32768 data: 8 7 7 7 7 7 7 6
121542024-02-13T21:44:10.419Z19/30 Write block 551 len 32768 data: 14 11 11 11 10 10 9 9
121552024-02-13T21:44:10.419Z20/30 Read block 89 len 8192
121562024-02-13T21:44:10.419Z21/30 Read block 372 len 16384
121572024-02-13T21:44:10.422Z22/30 Write block 372 len 20480 data: 12 11 9 11 10
121582024-02-13T21:44:10.422Z23/30 Read block 480 len 32768
121592024-02-13T21:44:10.422Z24/30 Read block 293 len 4096
121602024-02-13T21:44:10.426Z25/30 Write block 514 len 36864 data: 14 13 12 15 15 18 20 16 14
121612024-02-13T21:44:10.426Z26/30 Read block 574 len 12288
121622024-02-13T21:44:10.426Z27/30 Read block 296 len 36864
121632024-02-13T21:44:10.426Z28/30 Write block 250 len 12288 data: 9 11 9
121642024-02-13T21:44:10.431Z29/30 Read block 464 len 24576
121652024-02-13T21:44:10.431Z30/30 Write block 480 len 28672 data: 9 10 10 9 8 8 8
121662024-02-13T21:44:10.431Z----------------------------------------------------------------
121672024-02-13T21:44:10.431Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:29
121682024-02-13T21:44:10.431ZStates: Active Active Active
121692024-02-13T21:44:10.431ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121702024-02-13T21:44:10.431Z 2 Acked 1001 Write 5 Done Done Done false
121712024-02-13T21:44:10.432Z 3 Acked 1002 Read 1 Done Done Done false
121722024-02-13T21:44:10.432Z 4 Acked 1003 Write 8 Done Done Done false
121732024-02-13T21:44:10.432Z 5 Acked 1004 Read 4 Done Done Done false
121742024-02-13T21:44:10.432Z 6 Acked 1005 Write 9 Done Done Done false
121752024-02-13T21:44:10.432Z 7 Acked 1006 Write 6 Done Done Done false
121762024-02-13T21:44:10.432Z 8 Acked 1007 Write 10 Done Done Done false
121772024-02-13T21:44:10.432Z 9 Acked 1008 Write 7 Done Done Sent false
121782024-02-13T21:44:10.432Z 10 Acked 1009 Read 7 Done Done Sent false
121792024-02-13T21:44:10.432Z 11 Acked 1010 Read 6 Done Done Sent false
121802024-02-13T21:44:10.432Z 12 Acked 1011 Write 4 Done Done Sent false
121812024-02-13T21:44:10.432Z 13 Acked 1012 Read 1 Done Done Sent false
121822024-02-13T21:44:10.432Z 14 Acked 1013 Flush 0 Done Done Sent false
121832024-02-13T21:44:10.432Z 15 Acked 1014 Write 1 Done Done Sent false
121842024-02-13T21:44:10.432Z 16 Acked 1015 Write 6 Done Done Sent false
121852024-02-13T21:44:10.432Z 17 Acked 1016 Write 5 Done Done Sent false
121862024-02-13T21:44:10.432Z 18 Acked 1017 Write 8 Done Done Sent false
121872024-02-13T21:44:10.432Z 19 Acked 1018 Write 8 Done Done Sent false
121882024-02-13T21:44:10.432Z 20 Acked 1019 Read 2 Done Done Sent false
121892024-02-13T21:44:10.432Z 21 Acked 1020 Read 4 Done Done Sent false
121902024-02-13T21:44:10.432Z 22 Acked 1021 Write 5 Done Done Sent false
121912024-02-13T21:44:10.432Z 23 Acked 1022 Read 8 Done Done Sent false
121922024-02-13T21:44:10.432Z 24 Acked 1023 Read 1 Done Done Sent false
121932024-02-13T21:44:10.433Z 25 Acked 1024 Write 9 Done Done Sent false
121942024-02-13T21:44:10.433Z 26 Acked 1025 Read 3 Done Done Sent false
121952024-02-13T21:44:10.433Z 27 Acked 1026 Read 9 Done Done Sent false
121962024-02-13T21:44:10.433Z 28 Acked 1027 Write 3 Done Done Sent false
121972024-02-13T21:44:10.433Z 29 Acked 1028 Read 6 Done Done Sent false
121982024-02-13T21:44:10.433Z 30 Acked 1029 Write 7 Sent Sent Sent false
121992024-02-13T21:44:10.433Z STATES DS:0 DS:1 DS:2 TOTAL
122002024-02-13T21:44:10.433Z New 0 0 0 0
122012024-02-13T21:44:10.433Z Sent 1 1 22 24
122022024-02-13T21:44:10.433Z Done 28 28 7 63
122032024-02-13T21:44:10.433Z Skipped 0 0 0 0
122042024-02-13T21:44:10.433Z Error 0 0 0 0
122052024-02-13T21:44:10.433ZLast Flush: 1013 1013 1000
122062024-02-13T21:44:10.433ZDownstairs last five completed: 1000
122072024-02-13T21:44:10.433ZUpstairs last five completed: 30 29 28 27 26
122082024-02-13T21:44:10.433Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:09.172744121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1586}
122092024-02-13T21:44:10.433ZWrote out file "/tmp/test_repair_verify.data"
122102024-02-13T21:44:10.437Z
122112024-02-13T21:44:10.437ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
122122024-02-13T21:44:10.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122132024-02-13T21:44:10.454Z 0 000-019 104 104 104 193 193 193 F F F
122142024-02-13T21:44:10.454Z 1 020-039 112 112 107 207 207 198 F F T <---
122152024-02-13T21:44:10.454Z 2 040-059 108 108 108 201 201 201 F F F
122162024-02-13T21:44:10.454Z 3 060-079 110 110 110 203 203 203 F F F
122172024-02-13T21:44:10.454Z 4 080-099 104 104 104 193 193 193 F F F
122182024-02-13T21:44:10.454Z 5 100-119 110 110 110 204 204 204 F F F
122192024-02-13T21:44:10.454Z 6 120-139 106 106 106 195 195 195 F F F
122202024-02-13T21:44:10.454Z 7 140-159 109 109 109 202 202 202 F F F
122212024-02-13T21:44:10.455Z 8 160-179 108 108 108 201 201 201 F F F
122222024-02-13T21:44:10.455Z 9 180-199 110 110 110 204 204 204 F F F
122232024-02-13T21:44:10.455Z 10 200-219 110 110 110 203 203 203 F F F
122242024-02-13T21:44:10.458Z 11 220-239 110 110 110 203 203 203 F F F
122252024-02-13T21:44:10.458Z 12 240-259 110 110 110 204 204 204 T T F <---
122262024-02-13T21:44:10.458Z 13 260-279 111 111 111 205 205 205 F F F
122272024-02-13T21:44:10.458Z 14 280-299 106 106 106 197 197 197 F F F
122282024-02-13T21:44:10.458Z 15 300-319 112 112 104 207 207 190 F F T <---
122292024-02-13T21:44:10.458Z 16 320-339 112 112 106 207 207 197 F F T <---
122302024-02-13T21:44:10.458Z 17 340-359 111 111 111 205 205 205 F F F
122312024-02-13T21:44:10.458Z 18 360-379 111 111 111 205 205 205 T T F <---
122322024-02-13T21:44:10.458Z 19 380-399 112 112 108 207 207 200 T T T <---
122332024-02-13T21:44:10.458Z 20 400-419 112 112 109 207 207 202 F F T <---
122342024-02-13T21:44:10.458Z 21 420-439 109 109 109 202 202 202 F F F
122352024-02-13T21:44:10.458Z 22 440-459 110 110 110 203 203 203 F F F
122362024-02-13T21:44:10.458Z 23 460-479 110 110 110 203 203 203 T T F <---
122372024-02-13T21:44:10.458Z 24 480-499 108 108 108 200 200 200 T T F <---
122382024-02-13T21:44:10.458Z 25 500-519 110 110 110 203 203 203 T T F <---
122392024-02-13T21:44:10.458Z 26 520-539 108 108 108 200 200 200 T T F <---
122402024-02-13T21:44:10.459Z 27 540-559 112 112 108 207 207 200 T T F <---
122412024-02-13T21:44:10.459Z 28 560-579 108 108 108 199 199 199 F F F
122422024-02-13T21:44:10.459Z 29 580-599 110 110 110 204 204 204 F F F
122432024-02-13T21:44:10.459ZMax gen: 112, Max flush: 207
122442024-02-13T21:44:10.459ZError: Difference in extent metadata found!
122452024-02-13T21:44:10.459ZA Difference in extent metadata is expected here
122462024-02-13T21:44:10.459ZOn loop 56
122472024-02-13T21:44:10.459Z
122482024-02-13T21:44:10.462ZVerifying data now
122492024-02-13T21:44:16.073ZNo change after verify
122502024-02-13T21:44:16.073ZLoop: 56 Downstairs dump after verify (and repair):
122512024-02-13T21:44:16.087ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122522024-02-13T21:44:16.087Z 0 000-019 104 104 104 193 193 193 F F F
122532024-02-13T21:44:16.087Z 1 020-039 112 112 112 207 207 207 F F F
122542024-02-13T21:44:16.087Z 2 040-059 108 108 108 201 201 201 F F F
122552024-02-13T21:44:16.087Z 3 060-079 110 110 110 203 203 203 F F F
122562024-02-13T21:44:16.087Z 4 080-099 104 104 104 193 193 193 F F F
122572024-02-13T21:44:16.087Z 5 100-119 110 110 110 204 204 204 F F F
122582024-02-13T21:44:16.088Z 6 120-139 106 106 106 195 195 195 F F F
122592024-02-13T21:44:16.088Z 7 140-159 109 109 109 202 202 202 F F F
122602024-02-13T21:44:16.091Z 8 160-179 108 108 108 201 201 201 F F F
122612024-02-13T21:44:16.091Z 9 180-199 110 110 110 204 204 204 F F F
122622024-02-13T21:44:16.091Z 10 200-219 110 110 110 203 203 203 F F F
122632024-02-13T21:44:16.091Z 11 220-239 110 110 110 203 203 203 F F F
122642024-02-13T21:44:16.091Z 12 240-259 113 113 113 208 208 208 F F F
122652024-02-13T21:44:16.091Z 13 260-279 111 111 111 205 205 205 F F F
122662024-02-13T21:44:16.091Z 14 280-299 106 106 106 197 197 197 F F F
122672024-02-13T21:44:16.091Z 15 300-319 112 112 112 207 207 207 F F F
122682024-02-13T21:44:16.091Z 16 320-339 112 112 112 207 207 207 F F F
122692024-02-13T21:44:16.091Z 17 340-359 111 111 111 205 205 205 F F F
122702024-02-13T21:44:16.091Z 18 360-379 113 113 113 208 208 208 F F F
122712024-02-13T21:44:16.091Z 19 380-399 113 113 113 208 208 208 F F F
122722024-02-13T21:44:16.091Z 20 400-419 112 112 112 207 207 207 F F F
122732024-02-13T21:44:16.091Z 21 420-439 109 109 109 202 202 202 F F F
122742024-02-13T21:44:16.091Z 22 440-459 110 110 110 203 203 203 F F F
122752024-02-13T21:44:16.091Z 23 460-479 113 113 113 208 208 208 F F F
122762024-02-13T21:44:16.091Z 24 480-499 113 113 113 208 208 208 F F F
122772024-02-13T21:44:16.091Z 25 500-519 113 113 113 208 208 208 F F F
122782024-02-13T21:44:16.092Z 26 520-539 113 113 113 208 208 208 F F F
122792024-02-13T21:44:16.092Z 27 540-559 113 113 113 208 208 208 F F F
122802024-02-13T21:44:16.092Z 28 560-579 108 108 108 199 199 199 F F F
122812024-02-13T21:44:16.092Z 29 580-599 110 110 110 204 204 204 F F F
122822024-02-13T21:44:16.092ZMax gen: 113, Max flush: 208
122832024-02-13T21:44:16.092Z
122842024-02-13T21:44:16.092ZBegin loop 57
122852024-02-13T21:44:16.092ZDownstairs to restart: 0
122862024-02-13T21:44:16.108Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850519286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594}
122872024-02-13T21:44:16.113Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850827825Z","hostname":"Crucible runtime is spawned
122882024-02-13T21:44:16.113Zip-10-150-1-148.us-west-2.compute.internal","pid":1594}
122892024-02-13T21:44:16.113Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850875927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594}
122902024-02-13T21:44:16.113Z{"msg":"Crucible 6a345a7c-7e0a-4605-8d82-87d8e86265b0 has session id: 7469f8b2-84c3-450d-b994-b3d92246c2d2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850904857Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122912024-02-13T21:44:16.113Z{"msg":"Upstairs opts: Upstairs UUID: 6a345a7c-7e0a-4605-8d82-87d8e86265b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850946962Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122922024-02-13T21:44:16.113Z{"msg":"Crucible stats registered with UUID: 6a345a7c-7e0a-4605-8d82-87d8e86265b0","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.850993255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122932024-02-13T21:44:16.113Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851012867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594}
122942024-02-13T21:44:16.113Z{"msg":"6a345a7c-7e0a-4605-8d82-87d8e86265b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851047885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122952024-02-13T21:44:16.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851065399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122962024-02-13T21:44:16.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851102145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122972024-02-13T21:44:16.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851130115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122982024-02-13T21:44:16.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851157695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
122992024-02-13T21:44:16.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851184365Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123002024-02-13T21:44:16.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851213405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123012024-02-13T21:44:16.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851256969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123022024-02-13T21:44:16.114Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851321555Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123032024-02-13T21:44:16.114Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851373936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123042024-02-13T21:44:16.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851452038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123052024-02-13T21:44:16.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.851510067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123062024-02-13T21:44:16.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85152977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"io task","client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123072024-02-13T21:44:16.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852140629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123082024-02-13T21:44:16.114Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852215661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123092024-02-13T21:44:16.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852264054Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123102024-02-13T21:44:16.114Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852296742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123112024-02-13T21:44:16.114Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852780546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123122024-02-13T21:44:16.114Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.852921645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123132024-02-13T21:44:16.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85299131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123142024-02-13T21:44:16.114Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853071611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123152024-02-13T21:44:16.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853218747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123162024-02-13T21:44:16.114Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853262651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123172024-02-13T21:44:16.114Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853287622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123182024-02-13T21:44:16.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853534043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123192024-02-13T21:44:16.114Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853590333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123202024-02-13T21:44:16.114Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85362486Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123212024-02-13T21:44:16.114Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.853640154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123222024-02-13T21:44:16.119Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854049246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123232024-02-13T21:44:16.119Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854083734Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123242024-02-13T21:44:16.119Z{"msg":"[0]R flush_numbers[0..12]: [193, 207, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854099178Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123252024-02-13T21:44:16.119Z{"msg":"[0]R generation[0..12]: [104, 112, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854113803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123262024-02-13T21:44:16.119Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854135165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123272024-02-13T21:44:16.119Z{"msg":"[1]R flush_numbers[0..12]: [193, 207, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85414963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123282024-02-13T21:44:16.119Z{"msg":"[1]R generation[0..12]: [104, 112, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854170052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123292024-02-13T21:44:16.119Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854184407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123302024-02-13T21:44:16.119Z{"msg":"[2]R flush_numbers[0..12]: [193, 207, 201, 203, 193, 204, 195, 202, 201, 204, 203, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.8542052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123312024-02-13T21:44:16.119Z{"msg":"[2]R generation[0..12]: [104, 112, 108, 110, 104, 110, 106, 109, 108, 110, 110, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854219375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123322024-02-13T21:44:16.119ZWait for a query_work_queue command to finish before sending IO
123332024-02-13T21:44:16.119Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854245395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123342024-02-13T21:44:16.119Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85426022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123352024-02-13T21:44:16.119Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854280582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123362024-02-13T21:44:16.119Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854293907Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123372024-02-13T21:44:16.119Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854330584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"":"downstairs","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123382024-02-13T21:44:16.119Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854351157Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123392024-02-13T21:44:16.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854380506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"0","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123402024-02-13T21:44:16.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85439747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"1","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123412024-02-13T21:44:16.120Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854419352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"client":"2","":"downstairs","session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123422024-02-13T21:44:16.120Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854433257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123432024-02-13T21:44:16.120Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854456378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123442024-02-13T21:44:16.120Z{"msg":"6a345a7c-7e0a-4605-8d82-87d8e86265b0 is now active with session: 7469f8b2-84c3-450d-b994-b3d92246c2d2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.8544809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123452024-02-13T21:44:16.120Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.854501682Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594,"session_id":"7469f8b2-84c3-450d-b994-b3d92246c2d2"}
123462024-02-13T21:44:16.120Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:14.85453477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594}
123472024-02-13T21:44:16.120ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123482024-02-13T21:44:16.120ZLoading write count information from file "/tmp/test_repair_verify.data"
123492024-02-13T21:44:16.120ZRun Repair workload
123502024-02-13T21:44:16.120Z01/30 Write block 323 len 36864 data: 14 14 15 15 13 12 9 8 13
123512024-02-13T21:44:16.120Z02/30 Read block 419 len 20480
123522024-02-13T21:44:16.120Z03/30 Write block 240 len 32768 data: 11 10 10 9 8 7 7 7
123532024-02-13T21:44:16.120Z04/30 Flush
123542024-02-13T21:44:16.123Z05/30 Write block 28 len 32768 data: 6 8 7 10 11 11 12 14
123552024-02-13T21:44:16.123Z06/30 Write block 417 len 28672 data: 16 14 13 11 9 9 11
123562024-02-13T21:44:16.123Z07/30 Read block 511 len 24576
123572024-02-13T21:44:16.123Z08/30 Read block 312 len 20480
123582024-02-13T21:44:16.123Z09/30 Flush
123592024-02-13T21:44:16.126Z10/30 Write block 576 len 24576 data: 5 6 6 4 4 5
123602024-02-13T21:44:16.126Z11/30 Write block 570 len 8192 data: 8 5
123612024-02-13T21:44:16.129Z12/30 Write block 47 len 16384 data: 6 6 8 7
123622024-02-13T21:44:16.129Z13/30 Write block 87 len 24576 data: 6 6 6 6 7 8
123632024-02-13T21:44:16.129Z14/30 Read block 271 len 12288
123642024-02-13T21:44:16.129Z15/30 Read block 220 len 16384
123652024-02-13T21:44:16.132Z16/30 Read block 37 len 4096
123662024-02-13T21:44:16.132Z17/30 Read block 255 len 8192
123672024-02-13T21:44:16.132Z18/30 Read block 34 len 36864
123682024-02-13T21:44:16.133Z19/30 Read block 294 len 32768
123692024-02-13T21:44:16.135Z20/30 Read block 8 len 36864
123702024-02-13T21:44:16.135Z21/30 Write block 450 len 32768 data: 11 10 9 8 8 10 10 11
123712024-02-13T21:44:16.138Z22/30 Write block 292 len 16384 data: 11 13 12 10
123722024-02-13T21:44:16.138Z23/30 Read block 317 len 16384
123732024-02-13T21:44:16.138Z24/30 Read block 329 len 24576
123742024-02-13T21:44:16.138Z25/30 Write block 398 len 36864 data: 5 5 7 7 8 7 7 7 5
123752024-02-13T21:44:16.141Z26/30 Write block 477 len 4096 data: 10
123762024-02-13T21:44:16.141Z27/30 Write block 213 len 20480 data: 8 8 9 11 12
123772024-02-13T21:44:16.141Z28/30 Write block 506 len 28672 data: 10 10 10 11 12 13 12
123782024-02-13T21:44:16.141Z29/30 Read block 590 len 32768
123792024-02-13T21:44:16.145Z30/30 Read block 579 len 20480
123802024-02-13T21:44:16.145Z----------------------------------------------------------------
123812024-02-13T21:44:16.145Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123822024-02-13T21:44:16.145ZStates: Active Active Active
123832024-02-13T21:44:16.145ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123842024-02-13T21:44:16.145Z 1 Acked 1000 Write 9 Done Done Done false
123852024-02-13T21:44:16.145Z 2 Acked 1001 Read 5 Done Done Done false
123862024-02-13T21:44:16.145Z 3 Acked 1002 Write 8 Done Done Done false
123872024-02-13T21:44:16.145Z 4 Acked 1003 Flush 0 Sent Done Done false
123882024-02-13T21:44:16.145Z 5 Acked 1004 Write 8 Sent Done Done false
123892024-02-13T21:44:16.145Z 6 Acked 1005 Write 7 Sent Done Done false
123902024-02-13T21:44:16.145Z 7 Acked 1006 Read 6 Sent Done Done false
123912024-02-13T21:44:16.145Z 8 Acked 1007 Read 5 Sent Done Done false
123922024-02-13T21:44:16.145Z 9 Acked 1008 Flush 0 Sent Done Done false
123932024-02-13T21:44:16.145Z 10 Acked 1009 Write 6 Sent Done Done false
123942024-02-13T21:44:16.145Z 11 Acked 1010 Write 2 Sent Done Done false
123952024-02-13T21:44:16.146Z 12 Acked 1011 Write 4 Sent Done Done false
123962024-02-13T21:44:16.146Z 13 Acked 1012 Write 6 Sent Done Done false
123972024-02-13T21:44:16.146Z 14 Acked 1013 Read 3 Sent Done Done false
123982024-02-13T21:44:16.146Z 15 Acked 1014 Read 4 Sent Done Done false
123992024-02-13T21:44:16.146Z 16 Acked 1015 Read 1 Sent Done Done false
124002024-02-13T21:44:16.146Z 17 Acked 1016 Read 2 Sent Done Done false
124012024-02-13T21:44:16.146Z 18 Acked 1017 Read 9 Sent Done Done false
124022024-02-13T21:44:16.146Z 19 Acked 1018 Read 8 Sent Done Done false
124032024-02-13T21:44:16.146Z 20 Acked 1019 Read 9 Sent Done Done false
124042024-02-13T21:44:16.146Z 21 Acked 1020 Write 8 Sent Done Done false
124052024-02-13T21:44:16.146Z 22 Acked 1021 Write 4 Sent Done Done false
124062024-02-13T21:44:16.146Z 23 Acked 1022 Read 4 Sent Done Done false
124072024-02-13T21:44:16.146Z 24 Acked 1023 Read 6 Sent Done Done false
124082024-02-13T21:44:16.146Z 25 Acked 1024 Write 9 Sent Done Done false
124092024-02-13T21:44:16.146Z 26 Acked 1025 Write 1 Sent Done Done false
124102024-02-13T21:44:16.146Z 27 Acked 1026 Write 5 Sent Done Done false
124112024-02-13T21:44:16.146Z 28 Acked 1027 Write 7 Sent Done Done false
124122024-02-13T21:44:16.146Z 29 Acked 1028 Read 8 Sent Done Done false
124132024-02-13T21:44:16.146Z 30 Acked 1029 Read 5 Sent Done Done false
124142024-02-13T21:44:16.146Z STATES DS:0 DS:1 DS:2 TOTAL
124152024-02-13T21:44:16.146Z New 0 0 0 0
124162024-02-13T21:44:16.146Z Sent 27 0 0 27
124172024-02-13T21:44:16.146Z Done 3 30 30 63
124182024-02-13T21:44:16.146Z Skipped 0 0 0 0
124192024-02-13T21:44:16.146Z Error 0 0 0 0
124202024-02-13T21:44:16.147ZLast Flush: 0 1008 1008
124212024-02-13T21:44:16.147ZDownstairs last five completed:
124222024-02-13T21:44:16.147ZUpstairs last five completed: 30 29 28 27 26
124232024-02-13T21:44:16.147Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:14.885690519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1594}
124242024-02-13T21:44:16.147ZWrote out file "/tmp/test_repair_verify.data"
124252024-02-13T21:44:16.150Z
124262024-02-13T21:44:16.153ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124272024-02-13T21:44:16.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124282024-02-13T21:44:16.170Z 0 000-019 104 104 104 193 193 193 F F F
124292024-02-13T21:44:16.170Z 1 020-039 112 114 114 207 210 210 F F F <---
124302024-02-13T21:44:16.170Z 2 040-059 108 108 108 201 201 201 F T T <---
124312024-02-13T21:44:16.170Z 3 060-079 110 110 110 203 203 203 F F F
124322024-02-13T21:44:16.170Z 4 080-099 104 104 104 193 193 193 F T T <---
124332024-02-13T21:44:16.170Z 5 100-119 110 110 110 204 204 204 F F F
124342024-02-13T21:44:16.170Z 6 120-139 106 106 106 195 195 195 F F F
124352024-02-13T21:44:16.170Z 7 140-159 109 109 109 202 202 202 F F F
124362024-02-13T21:44:16.170Z 8 160-179 108 108 108 201 201 201 F F F
124372024-02-13T21:44:16.170Z 9 180-199 110 110 110 204 204 204 F F F
124382024-02-13T21:44:16.174Z 10 200-219 110 110 110 203 203 203 F T T <---
124392024-02-13T21:44:16.175Z 11 220-239 110 110 110 203 203 203 F F F
124402024-02-13T21:44:16.175Z 12 240-259 113 114 114 208 209 209 T F F <---
124412024-02-13T21:44:16.175Z 13 260-279 111 111 111 205 205 205 F F F
124422024-02-13T21:44:16.175Z 14 280-299 106 106 106 197 197 197 F T T <---
124432024-02-13T21:44:16.176Z 15 300-319 112 112 112 207 207 207 F F F
124442024-02-13T21:44:16.176Z 16 320-339 112 114 114 207 209 209 T F F <---
124452024-02-13T21:44:16.176Z 17 340-359 111 111 111 205 205 205 F F F
124462024-02-13T21:44:16.176Z 18 360-379 113 113 113 208 208 208 F F F
124472024-02-13T21:44:16.176Z 19 380-399 113 113 113 208 208 208 F T T <---
124482024-02-13T21:44:16.177Z 20 400-419 112 114 114 207 210 210 F T T <---
124492024-02-13T21:44:16.177Z 21 420-439 109 114 114 202 210 210 F F F <---
124502024-02-13T21:44:16.177Z 22 440-459 110 110 110 203 203 203 F T T <---
124512024-02-13T21:44:16.177Z 23 460-479 113 113 113 208 208 208 F T T <---
124522024-02-13T21:44:16.177Z 24 480-499 113 113 113 208 208 208 F F F
124532024-02-13T21:44:16.177Z 25 500-519 113 113 113 208 208 208 F T T <---
124542024-02-13T21:44:16.177Z 26 520-539 113 113 113 208 208 208 F F F
124552024-02-13T21:44:16.178Z 27 540-559 113 113 113 208 208 208 F F F
124562024-02-13T21:44:16.178Z 28 560-579 108 108 108 199 199 199 F T T <---
124572024-02-13T21:44:16.178Z 29 580-599 110 110 110 204 204 204 F T T <---
124582024-02-13T21:44:16.178ZMax gen: 114, Max flush: 210
124592024-02-13T21:44:16.178ZError: Difference in extent metadata found!
124602024-02-13T21:44:16.178ZA Difference in extent metadata is expected here
124612024-02-13T21:44:16.178ZOn loop 57
124622024-02-13T21:44:16.178Z
124632024-02-13T21:44:16.183ZVerifying data now
124642024-02-13T21:44:21.922ZNo change after verify
124652024-02-13T21:44:21.922ZLoop: 57 Downstairs dump after verify (and repair):
124662024-02-13T21:44:21.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124672024-02-13T21:44:21.937Z 0 000-019 104 104 104 193 193 193 F F F
124682024-02-13T21:44:21.937Z 1 020-039 114 114 114 210 210 210 F F F
124692024-02-13T21:44:21.937Z 2 040-059 115 115 115 211 211 211 F F F
124702024-02-13T21:44:21.937Z 3 060-079 110 110 110 203 203 203 F F F
124712024-02-13T21:44:21.937Z 4 080-099 115 115 115 211 211 211 F F F
124722024-02-13T21:44:21.938Z 5 100-119 110 110 110 204 204 204 F F F
124732024-02-13T21:44:21.938Z 6 120-139 106 106 106 195 195 195 F F F
124742024-02-13T21:44:21.938Z 7 140-159 109 109 109 202 202 202 F F F
124752024-02-13T21:44:21.949Z 8 160-179 108 108 108 201 201 201 F F F
124762024-02-13T21:44:21.949Z 9 180-199 110 110 110 204 204 204 F F F
124772024-02-13T21:44:21.950Z 10 200-219 115 115 115 211 211 211 F F F
124782024-02-13T21:44:21.950Z 11 220-239 110 110 110 203 203 203 F F F
124792024-02-13T21:44:21.950Z 12 240-259 114 114 114 209 209 209 F F F
124802024-02-13T21:44:21.950Z 13 260-279 111 111 111 205 205 205 F F F
124812024-02-13T21:44:21.950Z 14 280-299 115 115 115 211 211 211 F F F
124822024-02-13T21:44:21.950Z 15 300-319 112 112 112 207 207 207 F F F
124832024-02-13T21:44:21.950Z 16 320-339 114 114 114 209 209 209 F F F
124842024-02-13T21:44:21.950Z 17 340-359 111 111 111 205 205 205 F F F
124852024-02-13T21:44:21.951Z 18 360-379 113 113 113 208 208 208 F F F
124862024-02-13T21:44:21.951Z 19 380-399 115 115 115 211 211 211 F F F
124872024-02-13T21:44:21.951Z 20 400-419 115 115 115 211 211 211 F F F
124882024-02-13T21:44:21.951Z 21 420-439 114 114 114 210 210 210 F F F
124892024-02-13T21:44:21.951Z 22 440-459 115 115 115 211 211 211 F F F
124902024-02-13T21:44:21.951Z 23 460-479 115 115 115 211 211 211 F F F
124912024-02-13T21:44:21.951Z 24 480-499 113 113 113 208 208 208 F F F
124922024-02-13T21:44:21.952Z 25 500-519 115 115 115 211 211 211 F F F
124932024-02-13T21:44:21.952Z 26 520-539 113 113 113 208 208 208 F F F
124942024-02-13T21:44:21.952Z 27 540-559 113 113 113 208 208 208 F F F
124952024-02-13T21:44:21.952Z 28 560-579 115 115 115 211 211 211 F F F
124962024-02-13T21:44:21.952Z 29 580-599 115 115 115 211 211 211 F F F
124972024-02-13T21:44:21.953ZMax gen: 115, Max flush: 211
124982024-02-13T21:44:21.953Z
124992024-02-13T21:44:21.953ZBegin loop 58
125002024-02-13T21:44:21.953ZDownstairs to restart: 1
125012024-02-13T21:44:21.958Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698307463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
125022024-02-13T21:44:21.959ZCrucible runtime is spawned
125032024-02-13T21:44:21.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.6986752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
125042024-02-13T21:44:21.967Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698746094Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
125052024-02-13T21:44:21.967Z{"msg":"Crucible de9601b8-2ff8-42fd-8efb-ca8359cf5e9c has session id: 1e941ab6-8e82-4a69-a735-f9cad5ece841","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698775493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125062024-02-13T21:44:21.967Z{"msg":"Upstairs opts: Upstairs UUID: de9601b8-2ff8-42fd-8efb-ca8359cf5e9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698824235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125072024-02-13T21:44:21.967Z{"msg":"Crucible stats registered with UUID: de9601b8-2ff8-42fd-8efb-ca8359cf5e9c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698843639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125082024-02-13T21:44:21.967Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.69886835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
125092024-02-13T21:44:21.967Z{"msg":"de9601b8-2ff8-42fd-8efb-ca8359cf5e9c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698883354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125102024-02-13T21:44:21.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698904297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125112024-02-13T21:44:21.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.698956238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125122024-02-13T21:44:21.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699007109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125132024-02-13T21:44:21.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699037618Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125142024-02-13T21:44:21.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699064968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125152024-02-13T21:44:21.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699090549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125162024-02-13T21:44:21.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699139181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125172024-02-13T21:44:21.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699185175Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125182024-02-13T21:44:21.968Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699219932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125192024-02-13T21:44:21.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.69925208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125202024-02-13T21:44:21.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699292066Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125212024-02-13T21:44:21.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699321345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"io task","client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125222024-02-13T21:44:21.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699921928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125232024-02-13T21:44:21.969Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.699978327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125242024-02-13T21:44:21.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700019802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125252024-02-13T21:44:21.970Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700066165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125262024-02-13T21:44:21.970Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700506885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125272024-02-13T21:44:21.970Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700626302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125282024-02-13T21:44:21.970Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700805297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125292024-02-13T21:44:21.970Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.700835256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125302024-02-13T21:44:21.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701015551Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125312024-02-13T21:44:21.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701069721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125322024-02-13T21:44:21.970Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701097121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125332024-02-13T21:44:21.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701161568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125342024-02-13T21:44:21.971Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701197974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125352024-02-13T21:44:21.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701214019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125362024-02-13T21:44:21.971Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701418015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125372024-02-13T21:44:21.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701826387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125382024-02-13T21:44:21.971Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701860754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125392024-02-13T21:44:21.971Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 211, 203, 211, 204, 195, 202, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701877938Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125402024-02-13T21:44:21.971Z{"msg":"[0]R generation[0..12]: [104, 114, 115, 110, 115, 110, 106, 109, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701892403Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125412024-02-13T21:44:21.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701914455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125422024-02-13T21:44:21.972Z{"msg":"[1]R flush_numbers[0..12]: [193, 210, 211, 203, 211, 204, 195, 202, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701930049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125432024-02-13T21:44:21.972Z{"msg":"[1]R generation[0..12]: [104, 114, 115, 110, 115, 110, 106, 109, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701944654Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125442024-02-13T21:44:21.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701958769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125452024-02-13T21:44:21.972Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 211, 203, 211, 204, 195, 202, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701980281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125462024-02-13T21:44:21.972Z{"msg":"[2]R generation[0..12]: [104, 114, 115, 110, 115, 110, 106, 109, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.701994676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125472024-02-13T21:44:21.972Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.70200886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125482024-02-13T21:44:21.973Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702022136Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125492024-02-13T21:44:21.973Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702047596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125502024-02-13T21:44:21.973Z{"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702060901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125512024-02-13T21:44:21.973Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702074357Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"":"downstairs","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125522024-02-13T21:44:21.973Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
125532024-02-13T21:44:21.973Z,"time":"2024-02-13T21:44:20.702094779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125542024-02-13T21:44:21.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702115792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"0","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125552024-02-13T21:44:21.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702129607Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"1","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125562024-02-13T21:44:21.974Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702143342Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"client":"2","":"downstairs","session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125572024-02-13T21:44:21.974Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702163144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125582024-02-13T21:44:21.974Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.70217563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125592024-02-13T21:44:21.974Z{"msg":"de9601b8-2ff8-42fd-8efb-ca8359cf5e9c is now active with session: 1e941ab6-8e82-4a69-a735-f9cad5ece841","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702194933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125602024-02-13T21:44:21.975Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702206968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602,"session_id":"1e941ab6-8e82-4a69-a735-f9cad5ece841"}
125612024-02-13T21:44:21.975Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:20.702225992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
125622024-02-13T21:44:21.975ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125632024-02-13T21:44:21.975ZLoading write count information from file "/tmp/test_repair_verify.data"
125642024-02-13T21:44:21.975ZRun Repair workload
125652024-02-13T21:44:21.981Z01/30 Write block 134 len 36864 data: 9 10 9 7 11 9 10 11 14
125662024-02-13T21:44:21.981Z02/30 Write block 456 len 4096 data: 11
125672024-02-13T21:44:21.981Z03/30 Read block 226 len 20480
125682024-02-13T21:44:21.982Z04/30 Write block 287 len 36864 data: 6 9 10 9 10 12 14 13 11
125692024-02-13T21:44:21.982Z05/30 Read block 61 len 16384
125702024-02-13T21:44:21.982Z06/30 Read block 365 len 40960
125712024-02-13T21:44:21.982Z07/30 Read block 585 len 4096
125722024-02-13T21:44:21.982Z08/30 Read block 156 len 16384
125732024-02-13T21:44:21.982Z09/30 Read block 548 len 32768
125742024-02-13T21:44:21.982Z10/30 Read block 502 len 24576
125752024-02-13T21:44:21.982Z11/30 Write block 372 len 24576 data: 13 12 10 12 11 9
125762024-02-13T21:44:21.983Z12/30 Write block 396 len 28672 data: 7 6 6 6 8 8 9
125772024-02-13T21:44:21.983Z13/30 Write block 274 len 12288 data: 11 12 12
125782024-02-13T21:44:21.983Z14/30 Write block 540 len 8192 data: 13 13
125792024-02-13T21:44:21.983Z15/30 Read block 133 len 28672
125802024-02-13T21:44:21.983Z16/30 Read block 591 len 36864
125812024-02-13T21:44:21.983Z17/30 Write block 61 len 8192 data: 10 8
125822024-02-13T21:44:21.984Z18/30 Flush
125832024-02-13T21:44:21.988Z19/30 Flush
125842024-02-13T21:44:21.988Z20/30 Write block 440 len 16384 data: 8 9 9 8
125852024-02-13T21:44:21.988Z21/30 Read block 584 len 12288
125862024-02-13T21:44:21.988Z22/30 Read block 582 len 8192
125872024-02-13T21:44:21.988Z23/30 Read block 133 len 4096
125882024-02-13T21:44:21.988Z24/30 Write block 142 len 12288 data: 15 12 13
125892024-02-13T21:44:21.989Z25/30 Flush
125902024-02-13T21:44:21.993Z26/30 Write block 356 len 12288 data: 14 13 12
125912024-02-13T21:44:21.993Z27/30 Write block 145 len 16384 data: 13 10 9 6
125922024-02-13T21:44:21.993Z28/30 Read block 357 len 20480
125932024-02-13T21:44:21.993Z29/30 Read block 344 len 16384
125942024-02-13T21:44:21.993Z30/30 Read block 231 len 40960
125952024-02-13T21:44:21.998Z----------------------------------------------------------------
125962024-02-13T21:44:21.998Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125972024-02-13T21:44:21.998ZStates: Active Active Active
125982024-02-13T21:44:21.998ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125992024-02-13T21:44:21.998Z 1 Acked 1000 Write 9 Done Done Done false
126002024-02-13T21:44:21.999Z 2 Acked 1001 Write 1 Done Done Done false
126012024-02-13T21:44:21.999Z 3 Acked 1002 Read 5 Done Done Done false
126022024-02-13T21:44:21.999Z 4 Acked 1003 Write 9 Done Done Done false
126032024-02-13T21:44:21.999Z 5 Acked 1004 Read 4 Done Done Done false
126042024-02-13T21:44:21.999Z 6 Acked 1005 Read 10 Done Sent Done false
126052024-02-13T21:44:21.999Z 7 Acked 1006 Read 1 Done Sent Done false
126062024-02-13T21:44:21.999Z 8 Acked 1007 Read 4 Done Sent Done false
126072024-02-13T21:44:21.999Z 9 Acked 1008 Read 8 Done Sent Done false
126082024-02-13T21:44:22.000Z 10 Acked 1009 Read 6 Done Sent Done false
126092024-02-13T21:44:22.000Z 11 Acked 1010 Write 6 Done Sent Done false
126102024-02-13T21:44:22.000Z 12 Acked 1011 Write 7 Done Sent Done false
126112024-02-13T21:44:22.000Z 13 Acked 1012 Write 3 Done Sent Done false
126122024-02-13T21:44:22.002Z 14 Acked 1013 Write 2 Done Sent Done false
126132024-02-13T21:44:22.002Z 15 Acked 1014 Read 7 Done Sent Done false
126142024-02-13T21:44:22.002Z 16 Acked 1015 Read 9 Done Sent Done false
126152024-02-13T21:44:22.003Z 17 Acked 1016 Write 2 Done Sent Done false
126162024-02-13T21:44:22.003Z 18 Acked 1017 Flush 0 Done Sent Done false
126172024-02-13T21:44:22.003Z 19 Acked 1018 Flush 0 Done Sent Done false
126182024-02-13T21:44:22.003Z 20 Acked 1019 Write 4 Done Sent Done false
126192024-02-13T21:44:22.003Z 21 Acked 1020 Read 3 Done Sent Done false
126202024-02-13T21:44:22.003Z 22 Acked 1021 Read 2 Done Sent Done false
126212024-02-13T21:44:22.003Z 23 Acked 1022 Read 1 Done Sent Done false
126222024-02-13T21:44:22.003Z 24 Acked 1023 Write 3 Done Sent Done false
126232024-02-13T21:44:22.003Z 25 Acked 1024 Flush 0 Done Sent Done false
126242024-02-13T21:44:22.003Z 26 Acked 1025 Write 3 Done Sent Done false
126252024-02-13T21:44:22.003Z 27 Acked 1026 Write 4 Done Sent Done false
126262024-02-13T21:44:22.003Z 28 Acked 1027 Read 5 Done Sent Done false
126272024-02-13T21:44:22.004Z 29 Acked 1028 Read 4 Done Sent Done false
126282024-02-13T21:44:22.004Z 30 Acked 1029 Read 10 Done Sent Done false
126292024-02-13T21:44:22.004Z STATES DS:0 DS:1 DS:2 TOTAL
126302024-02-13T21:44:22.004Z New 0 0 0 0
126312024-02-13T21:44:22.004Z Sent 0 25 0 25
126322024-02-13T21:44:22.004Z Done 30 5 30 65
126332024-02-13T21:44:22.004Z Skipped 0 0 0 0
126342024-02-13T21:44:22.004Z Error 0 0 0 0
126352024-02-13T21:44:22.004ZLast Flush: 1024 0 1024
126362024-02-13T21:44:22.004ZDownstairs last five completed:
126372024-02-13T21:44:22.004ZUpstairs last five completed: 30 29 28 27 26
126382024-02-13T21:44:22.004Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:20.73280347Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1602}
126392024-02-13T21:44:22.005ZWrote out file "/tmp/test_repair_verify.data"
126402024-02-13T21:44:22.005Z
126412024-02-13T21:44:22.009ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126422024-02-13T21:44:22.033ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126432024-02-13T21:44:22.033Z 0 000-019 104 104 104 193 193 193 F F F
126442024-02-13T21:44:22.033Z 1 020-039 114 114 114 210 210 210 F F F
126452024-02-13T21:44:22.033Z 2 040-059 115 115 115 211 211 211 F F F
126462024-02-13T21:44:22.033Z 3 060-079 116 110 116 212 203 212 F F F <---
126472024-02-13T21:44:22.033Z 4 080-099 115 115 115 211 211 211 F F F
126482024-02-13T21:44:22.033Z 5 100-119 110 110 110 204 204 204 F F F
126492024-02-13T21:44:22.033Z 6 120-139 116 106 116 212 195 212 F T F <---
126502024-02-13T21:44:22.034Z 7 140-159 116 109 116 214 202 214 T T T <---
126512024-02-13T21:44:22.034Z 8 160-179 108 108 108 201 201 201 F F F
126522024-02-13T21:44:22.034Z 9 180-199 110 110 110 204 204 204 F F F
126532024-02-13T21:44:22.034Z 10 200-219 115 115 115 211 211 211 F F F
126542024-02-13T21:44:22.034Z 11 220-239 110 110 110 203 203 203 F F F
126552024-02-13T21:44:22.034Z 12 240-259 114 114 114 209 209 209 F F F
126562024-02-13T21:44:22.034Z 13 260-279 116 111 116 212 205 212 F F F <---
126572024-02-13T21:44:22.034Z 14 280-299 116 115 116 212 211 212 F T F <---
126582024-02-13T21:44:22.034Z 15 300-319 112 112 112 207 207 207 F F F
126592024-02-13T21:44:22.034Z 16 320-339 114 114 114 209 209 209 F F F
126602024-02-13T21:44:22.035Z 17 340-359 111 111 111 205 205 205 T F T <---
126612024-02-13T21:44:22.035Z 18 360-379 116 113 116 212 208 212 F F F <---
126622024-02-13T21:44:22.035Z 19 380-399 116 115 116 212 211 212 F F F <---
126632024-02-13T21:44:22.035Z 20 400-419 116 115 116 212 211 212 F F F <---
126642024-02-13T21:44:22.035Z 21 420-439 114 114 114 210 210 210 F F F
126652024-02-13T21:44:22.035Z 22 440-459 116 115 116 214 211 214 F T F <---
126662024-02-13T21:44:22.035Z 23 460-479 115 115 115 211 211 211 F F F
126672024-02-13T21:44:22.035Z 24 480-499 113 113 113 208 208 208 F F F
126682024-02-13T21:44:22.035Z 25 500-519 115 115 115 211 211 211 F F F
126692024-02-13T21:44:22.036Z 26 520-539 113 113 113 208 208 208 F F F
126702024-02-13T21:44:22.036Z 27 540-559 116 113 116 212 208 212 F F F <---
126712024-02-13T21:44:22.036Z 28 560-579 115 115 115 211 211 211 F F F
126722024-02-13T21:44:22.036Z 29 580-599 115 115 115 211 211 211 F F F
126732024-02-13T21:44:22.036ZMax gen: 116, Max flush: 214
126742024-02-13T21:44:22.036ZError: Difference in extent metadata found!
126752024-02-13T21:44:22.036ZA Difference in extent metadata is expected here
126762024-02-13T21:44:22.036ZOn loop 58
126772024-02-13T21:44:22.036Z
126782024-02-13T21:44:22.036ZVerifying data now
126792024-02-13T21:44:27.598ZNo change after verify
126802024-02-13T21:44:27.598ZLoop: 58 Downstairs dump after verify (and repair):
126812024-02-13T21:44:27.613ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126822024-02-13T21:44:27.613Z 0 000-019 104 104 104 193 193 193 F F F
126832024-02-13T21:44:27.613Z 1 020-039 114 114 114 210 210 210 F F F
126842024-02-13T21:44:27.613Z 2 040-059 115 115 115 211 211 211 F F F
126852024-02-13T21:44:27.613Z 3 060-079 116 116 116 212 212 212 F F F
126862024-02-13T21:44:27.614Z 4 080-099 115 115 115 211 211 211 F F F
126872024-02-13T21:44:27.614Z 5 100-119 110 110 110 204 204 204 F F F
126882024-02-13T21:44:27.614Z 6 120-139 116 116 116 212 212 212 F F F
126892024-02-13T21:44:27.614Z 7 140-159 117 117 117 215 215 215 F F F
126902024-02-13T21:44:27.614Z 8 160-179 108 108 108 201 201 201 F F F
126912024-02-13T21:44:27.614Z 9 180-199 110 110 110 204 204 204 F F F
126922024-02-13T21:44:27.615Z 10 200-219 115 115 115 211 211 211 F F F
126932024-02-13T21:44:27.615Z 11 220-239 110 110 110 203 203 203 F F F
126942024-02-13T21:44:27.619Z 12 240-259 114 114 114 209 209 209 F F F
126952024-02-13T21:44:27.619Z 13 260-279 116 116 116 212 212 212 F F F
126962024-02-13T21:44:27.620Z 14 280-299 116 116 116 212 212 212 F F F
126972024-02-13T21:44:27.620Z 15 300-319 112 112 112 207 207 207 F F F
126982024-02-13T21:44:27.620Z 16 320-339 114 114 114 209 209 209 F F F
126992024-02-13T21:44:27.620Z 17 340-359 117 117 117 215 215 215 F F F
127002024-02-13T21:44:27.621Z 18 360-379 116 116 116 212 212 212 F F F
127012024-02-13T21:44:27.621Z 19 380-399 116 116 116 212 212 212 F F F
127022024-02-13T21:44:27.621Z 20 400-419 116 116 116 212 212 212 F F F
127032024-02-13T21:44:27.621Z 21 420-439 114 114 114 210 210 210 F F F
127042024-02-13T21:44:27.621Z 22 440-459 116 116 116 214 214 214 F F F
127052024-02-13T21:44:27.621Z 23 460-479 115 115 115 211 211 211 F F F
127062024-02-13T21:44:27.621Z 24 480-499 113 113 113 208 208 208 F F F
127072024-02-13T21:44:27.622Z 25 500-519 115 115 115 211 211 211 F F F
127082024-02-13T21:44:27.622Z 26 520-539 113 113 113 208 208 208 F F F
127092024-02-13T21:44:27.622Z 27 540-559 116 116 116 212 212 212 F F F
127102024-02-13T21:44:27.622Z 28 560-579 115 115 115 211 211 211 F F F
127112024-02-13T21:44:27.622Z 29 580-599 115 115 115 211 211 211 F F F
127122024-02-13T21:44:27.622ZMax gen: 117, Max flush: 215
127132024-02-13T21:44:27.622Z
127142024-02-13T21:44:27.622ZBegin loop 59
127152024-02-13T21:44:27.623ZDownstairs to restart: 0
127162024-02-13T21:44:27.634Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.3716167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
127172024-02-13T21:44:27.634Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"Crucible runtime is spawned
127182024-02-13T21:44:27.635Z2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.371904076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
127192024-02-13T21:44:27.635Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.371935964Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
127202024-02-13T21:44:27.642Z{"msg":"Crucible 2b60cad5-ebc6-4509-aa81-8639d12e672f has session id: d570f658-0b1f-4b18-8367-ce5d761d75d0","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.371961775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127212024-02-13T21:44:27.642Z{"msg":"Upstairs opts: Upstairs UUID: 2b60cad5-ebc6-4509-aa81-8639d12e672f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372012496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127222024-02-13T21:44:27.642Z{"msg":"Crucible stats registered with UUID: 2b60cad5-ebc6-4509-aa81-8639d12e672f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37205924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127232024-02-13T21:44:27.642Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372100234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
127242024-02-13T21:44:27.642Z{"msg":"2b60cad5-ebc6-4509-aa81-8639d12e672f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372116979Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127252024-02-13T21:44:27.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37214187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127262024-02-13T21:44:27.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372160723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127272024-02-13T21:44:27.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372187853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127282024-02-13T21:44:27.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372206886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127292024-02-13T21:44:27.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372255408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127302024-02-13T21:44:27.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372283088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127312024-02-13T21:44:27.643Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372311048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127322024-02-13T21:44:27.643Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372376574Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127332024-02-13T21:44:27.644Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372408043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127342024-02-13T21:44:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372437672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127352024-02-13T21:44:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372464992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127362024-02-13T21:44:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.372509026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"io task","client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127372024-02-13T21:44:27.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.373205973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127382024-02-13T21:44:27.644Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37324293Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127392024-02-13T21:44:27.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.373258534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127402024-02-13T21:44:27.644Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.373290043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127412024-02-13T21:44:27.644Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.373781465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127422024-02-13T21:44:27.645Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.373919325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127432024-02-13T21:44:27.645Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37420704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127442024-02-13T21:44:27.645Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374257042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127452024-02-13T21:44:27.645Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374333604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127462024-02-13T21:44:27.645Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374381567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127472024-02-13T21:44:27.645Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374426051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127482024-02-13T21:44:27.645Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37445582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127492024-02-13T21:44:27.646Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374485189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127502024-02-13T21:44:27.646Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374528404Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127512024-02-13T21:44:27.646Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.374707419Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127522024-02-13T21:44:27.646Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375134244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127532024-02-13T21:44:27.646Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375169651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127542024-02-13T21:44:27.646Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 211, 212, 211, 204, 212, 215, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375186615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127552024-02-13T21:44:27.646Z{"msg":"[0]R generation[0..12]: [104, 114, 115, 116, 115, 110, 116, 117, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375253041Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127562024-02-13T21:44:27.646Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375289718Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127572024-02-13T21:44:27.647Z{"Wait for a query_work_queue command to finish before sending IO
127582024-02-13T21:44:27.647Zmsg":"[1]R flush_numbers[0..12]: [193, 210, 211, 212, 211, 204, 212, 215, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375319587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127592024-02-13T21:44:27.647Z{"msg":"[1]R generation[0..12]: [104, 114, 115, 116, 115, 110, 116, 117, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375352465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127602024-02-13T21:44:27.647Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375378645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127612024-02-13T21:44:27.647Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 211, 212, 211, 204, 212, 215, 201, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375410954Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127622024-02-13T21:44:27.647Z{"msg":"[2]R generation[0..12]: [104, 114, 115, 116, 115, 110, 116, 117, 108, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375436784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127632024-02-13T21:44:27.647Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375469532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127642024-02-13T21:44:27.647Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375493763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127652024-02-13T21:44:27.648Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375525342Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127662024-02-13T21:44:27.648Z{"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375549773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127672024-02-13T21:44:27.648Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375572405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"":"downstairs","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127682024-02-13T21:44:27.648Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375603294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127692024-02-13T21:44:27.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375624756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"0","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127702024-02-13T21:44:27.648Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375651866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"1","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127712024-02-13T21:44:27.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375675228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"client":"2","":"downstairs","session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127722024-02-13T21:44:27.649Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375702348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127732024-02-13T21:44:27.649Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.37572378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127742024-02-13T21:44:27.649Z{"msg":"2b60cad5-ebc6-4509-aa81-8639d12e672f is now active with session: d570f658-0b1f-4b18-8367-ce5d761d75d0","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.3757518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127752024-02-13T21:44:27.649Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375772802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610,"session_id":"d570f658-0b1f-4b18-8367-ce5d761d75d0"}
127762024-02-13T21:44:27.649Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:26.375798473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
127772024-02-13T21:44:27.649ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127782024-02-13T21:44:27.649ZLoading write count information from file "/tmp/test_repair_verify.data"
127792024-02-13T21:44:27.649ZRun Repair workload
127802024-02-13T21:44:27.650Z01/30 Flush
127812024-02-13T21:44:27.654Z02/30 Write block 443 len 8192 data: 9 8
127822024-02-13T21:44:27.654Z03/30 Flush
127832024-02-13T21:44:27.654Z04/30 Write block 329 len 8192 data: 10 9
127842024-02-13T21:44:27.655Z05/30 Flush
127852024-02-13T21:44:27.655Z06/30 Flush
127862024-02-13T21:44:27.655Z07/30 Write block 315 len 36864 data: 9 11 12 12 14 13 12 13 15
127872024-02-13T21:44:27.655Z08/30 Read block 11 len 36864
127882024-02-13T21:44:27.655Z09/30 Read block 446 len 20480
127892024-02-13T21:44:27.655Z10/30 Write block 437 len 20480 data: 8 10 9 9 10
127902024-02-13T21:44:27.655Z11/30 Write block 527 len 40960 data: 7 8 7 7 6 7 7 7 6 5
127912024-02-13T21:44:27.656Z12/30 Write block 241 len 20480 data: 11 11 10 9 8
127922024-02-13T21:44:27.656Z13/30 Read block 521 len 4096
127932024-02-13T21:44:27.656Z14/30 Write block 59 len 20480 data: 13 11 11 9 7
127942024-02-13T21:44:27.656Z15/30 Write block 397 len 16384 data: 7 7 7 9
127952024-02-13T21:44:27.656Z16/30 Read block 430 len 24576
127962024-02-13T21:44:27.660Z17/30 Write block 148 len 32768 data: 7 7 7 7 6 8 8 9
127972024-02-13T21:44:27.660Z18/30 Write block 120 len 32768 data: 8 9 10 10 8 13 13 12
127982024-02-13T21:44:27.660Z19/30 Read block 67 len 4096
127992024-02-13T21:44:27.660Z20/30 Read block 91 len 8192
128002024-02-13T21:44:27.660Z21/30 Write block 171 len 24576 data: 4 7 6 5 6 5
128012024-02-13T21:44:27.660Z22/30 Write block 330 len 12288 data: 10 14 14
128022024-02-13T21:44:27.661Z23/30 Flush
128032024-02-13T21:44:27.666Z24/30 Write block 531 len 20480 data: 7 8 8 8 7
128042024-02-13T21:44:27.669Z25/30 Flush
128052024-02-13T21:44:27.670Z26/30 Write block 85 len 20480 data: 9 8 7 7 7
128062024-02-13T21:44:27.670Z27/30 Write block 300 len 36864 data: 7 6 6 8 8 7 6 9 7
128072024-02-13T21:44:27.675Z28/30 Read block 430 len 40960
128082024-02-13T21:44:27.675Z29/30 Write block 80 len 20480 data: 7 9 9 9 8
128092024-02-13T21:44:27.675Z30/30 Read block 314 len 40960
128102024-02-13T21:44:27.675Z----------------------------------------------------------------
128112024-02-13T21:44:27.675Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:24
128122024-02-13T21:44:27.675ZStates: Active Active Active
128132024-02-13T21:44:27.675ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
128142024-02-13T21:44:27.675Z 7 Acked 1006 Write 9 Done Done Done false
128152024-02-13T21:44:27.676Z 8 Acked 1007 Read 9 Done Done Done false
128162024-02-13T21:44:27.676Z 9 Acked 1008 Read 5 Done Done Done false
128172024-02-13T21:44:27.676Z 10 Acked 1009 Write 5 Sent Done Done false
128182024-02-13T21:44:27.676Z 11 Acked 1010 Write 10 Sent Done Done false
128192024-02-13T21:44:27.676Z 12 Acked 1011 Write 5 Sent Done Done false
128202024-02-13T21:44:27.676Z 13 Acked 1012 Read 1 Sent Done Done false
128212024-02-13T21:44:27.676Z 14 Acked 1013 Write 5 Sent Done Done false
128222024-02-13T21:44:27.676Z 15 Acked 1014 Write 4 Sent Done Done false
128232024-02-13T21:44:27.677Z 16 Acked 1015 Read 6 Sent Done Done false
128242024-02-13T21:44:27.677Z 17 Acked 1016 Write 8 Sent Done Done false
128252024-02-13T21:44:27.677Z 18 Acked 1017 Write 8 Sent Done Done false
128262024-02-13T21:44:27.677Z 19 Acked 1018 Read 1 Sent Done Done false
128272024-02-13T21:44:27.677Z 20 Acked 1019 Read 2 Sent Done Done false
128282024-02-13T21:44:27.677Z 21 Acked 1020 Write 6 Sent Done Done false
128292024-02-13T21:44:27.678Z 22 Acked 1021 Write 3 Sent Done Done false
128302024-02-13T21:44:27.678Z 23 Acked 1022 Flush 0 Sent Done Done false
128312024-02-13T21:44:27.678Z 24 Acked 1023 Write 5 Sent Done Done false
128322024-02-13T21:44:27.678Z 25 Acked 1024 Flush 0 Sent Done Done false
128332024-02-13T21:44:27.678Z 26 Acked 1025 Write 5 Sent Done Done false
128342024-02-13T21:44:27.678Z 27 Acked 1026 Write 9 Sent Done Done false
128352024-02-13T21:44:27.678Z 28 Acked 1027 Read 10 Sent Done Done false
128362024-02-13T21:44:27.679Z 29 Acked 1028 Write 5 Sent Done Done false
128372024-02-13T21:44:27.679Z 30 Acked 1029 Read 10 Sent Sent Done false
128382024-02-13T21:44:27.679Z STATES DS:0 DS:1 DS:2 TOTAL
128392024-02-13T21:44:27.679Z New 0 0 0 0
128402024-02-13T21:44:27.679Z Sent 21 1 0 22
128412024-02-13T21:44:27.679Z Done 3 23 24 50
128422024-02-13T21:44:27.679Z Skipped 0 0 0 0
128432024-02-13T21:44:27.679Z Error 0 0 0 0
128442024-02-13T21:44:27.680ZLast Flush: 1005 1024 1024
128452024-02-13T21:44:27.680ZDownstairs last five completed: 1005 1004 1003 1002 1001
128462024-02-13T21:44:27.680ZUpstairs last five completed: 30 29 28 27 26
128472024-02-13T21:44:27.680Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:26.411382036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1610}
128482024-02-13T21:44:27.680ZWrote out file "/tmp/test_repair_verify.data"
128492024-02-13T21:44:27.684Z
128502024-02-13T21:44:27.684ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128512024-02-13T21:44:27.700ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128522024-02-13T21:44:27.700Z 0 000-019 104 104 104 193 193 193 F F F
128532024-02-13T21:44:27.700Z 1 020-039 114 114 114 210 210 210 F F F
128542024-02-13T21:44:27.700Z 2 040-059 115 118 118 211 220 220 F F F <---
128552024-02-13T21:44:27.701Z 3 060-079 116 118 118 212 220 220 F F F <---
128562024-02-13T21:44:27.701Z 4 080-099 115 115 115 211 211 211 F T T <---
128572024-02-13T21:44:27.701Z 5 100-119 110 110 110 204 204 204 F F F
128582024-02-13T21:44:27.701Z 6 120-139 116 118 118 212 220 220 F F F <---
128592024-02-13T21:44:27.701Z 7 140-159 117 118 118 215 220 220 F F F <---
128602024-02-13T21:44:27.701Z 8 160-179 108 118 118 201 220 220 F F F <---
128612024-02-13T21:44:27.711Z 9 180-199 110 110 110 204 204 204 F F F
128622024-02-13T21:44:27.711Z 10 200-219 115 115 115 211 211 211 F F F
128632024-02-13T21:44:27.711Z 11 220-239 110 110 110 203 203 203 F F F
128642024-02-13T21:44:27.711Z 12 240-259 114 118 118 209 220 220 F F F <---
128652024-02-13T21:44:27.711Z 13 260-279 116 116 116 212 212 212 F F F
128662024-02-13T21:44:27.711Z 14 280-299 116 116 116 212 212 212 F F F
128672024-02-13T21:44:27.711Z 15 300-319 112 118 118 207 220 220 T T T <---
128682024-02-13T21:44:27.712Z 16 320-339 118 118 118 218 220 220 T F F <---
128692024-02-13T21:44:27.712Z 17 340-359 117 117 117 215 215 215 F F F
128702024-02-13T21:44:27.712Z 18 360-379 116 116 116 212 212 212 F F F
128712024-02-13T21:44:27.712Z 19 380-399 116 118 118 212 220 220 F F F <---
128722024-02-13T21:44:27.712Z 20 400-419 116 118 118 212 220 220 F F F <---
128732024-02-13T21:44:27.712Z 21 420-439 114 118 118 210 220 220 F F F <---
128742024-02-13T21:44:27.712Z 22 440-459 118 118 118 217 220 220 F F F <---
128752024-02-13T21:44:27.712Z 23 460-479 115 115 115 211 211 211 F F F
128762024-02-13T21:44:27.712Z 24 480-499 113 113 113 208 208 208 F F F
128772024-02-13T21:44:27.712Z 25 500-519 115 115 115 211 211 211 F F F
128782024-02-13T21:44:27.712Z 26 520-539 113 118 118 208 221 221 F F F <---
128792024-02-13T21:44:27.713Z 27 540-559 116 116 116 212 212 212 F F F
128802024-02-13T21:44:27.713Z 28 560-579 115 115 115 211 211 211 F F F
128812024-02-13T21:44:27.713Z 29 580-599 115 115 115 211 211 211 F F F
128822024-02-13T21:44:27.713ZMax gen: 118, Max flush: 221
128832024-02-13T21:44:27.713ZError: Difference in extent metadata found!
128842024-02-13T21:44:27.713ZA Difference in extent metadata is expected here
128852024-02-13T21:44:27.714ZOn loop 59
128862024-02-13T21:44:27.714Z
128872024-02-13T21:44:27.714ZVerifying data now
128882024-02-13T21:44:32.370ZNo change after verify
128892024-02-13T21:44:32.370ZLoop: 59 Downstairs dump after verify (and repair):
128902024-02-13T21:44:32.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128912024-02-13T21:44:32.385Z 0 000-019 104 104 104 193 193 193 F F F
128922024-02-13T21:44:32.385Z 1 020-039 114 114 114 210 210 210 F F F
128932024-02-13T21:44:32.385Z 2 040-059 118 118 118 220 220 220 F F F
128942024-02-13T21:44:32.385Z 3 060-079 118 118 118 220 220 220 F F F
128952024-02-13T21:44:32.385Z 4 080-099 119 119 119 222 222 222 F F F
128962024-02-13T21:44:32.385Z 5 100-119 110 110 110 204 204 204 F F F
128972024-02-13T21:44:32.385Z 6 120-139 118 118 118 220 220 220 F F F
128982024-02-13T21:44:32.385Z 7 140-159 118 118 118 220 220 220 F F F
128992024-02-13T21:44:32.385Z 8 160-179 118 118 118 220 220 220 F F F
129002024-02-13T21:44:32.385Z 9 180-199 110 110 110 204 204 204 F F F
129012024-02-13T21:44:32.385Z 10 200-219 115 115 115 211 211 211 F F F
129022024-02-13T21:44:32.388Z 11 220-239 110 110 110 203 203 203 F F F
129032024-02-13T21:44:32.388Z 12 240-259 118 118 118 220 220 220 F F F
129042024-02-13T21:44:32.389Z 13 260-279 116 116 116 212 212 212 F F F
129052024-02-13T21:44:32.389Z 14 280-299 116 116 116 212 212 212 F F F
129062024-02-13T21:44:32.389Z 15 300-319 119 119 119 222 222 222 F F F
129072024-02-13T21:44:32.389Z 16 320-339 118 118 118 220 220 220 F F F
129082024-02-13T21:44:32.389Z 17 340-359 117 117 117 215 215 215 F F F
129092024-02-13T21:44:32.389Z 18 360-379 116 116 116 212 212 212 F F F
129102024-02-13T21:44:32.389Z 19 380-399 118 118 118 220 220 220 F F F
129112024-02-13T21:44:32.389Z 20 400-419 118 118 118 220 220 220 F F F
129122024-02-13T21:44:32.389Z 21 420-439 118 118 118 220 220 220 F F F
129132024-02-13T21:44:32.389Z 22 440-459 118 118 118 220 220 220 F F F
129142024-02-13T21:44:32.389Z 23 460-479 115 115 115 211 211 211 F F F
129152024-02-13T21:44:32.389Z 24 480-499 113 113 113 208 208 208 F F F
129162024-02-13T21:44:32.389Z 25 500-519 115 115 115 211 211 211 F F F
129172024-02-13T21:44:32.389Z 26 520-539 118 118 118 221 221 221 F F F
129182024-02-13T21:44:32.389Z 27 540-559 116 116 116 212 212 212 F F F
129192024-02-13T21:44:32.389Z 28 560-579 115 115 115 211 211 211 F F F
129202024-02-13T21:44:32.389Z 29 580-599 115 115 115 211 211 211 F F F
129212024-02-13T21:44:32.389ZMax gen: 119, Max flush: 222
129222024-02-13T21:44:32.389Z
129232024-02-13T21:44:32.389ZBegin loop 60
129242024-02-13T21:44:32.389ZDownstairs to restart: 2
129252024-02-13T21:44:32.406Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142406441Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
129262024-02-13T21:44:32.412Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142669656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
129272024-02-13T21:44:32.412Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142712111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
129282024-02-13T21:44:32.412Z{"msg":"Crucible c92ba656-1c39-47c9-9884-49deb2c91857 has session id: 36bba27b-ea4c-4d0e-b770-db1745ce7106","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142736931Z","hostname":"Crucible runtime is spawned
129292024-02-13T21:44:32.413Zip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129302024-02-13T21:44:32.413Z{"msg":"Upstairs opts: Upstairs UUID: c92ba656-1c39-47c9-9884-49deb2c91857, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142792731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129312024-02-13T21:44:32.413Z{"msg":"Crucible stats registered with UUID: c92ba656-1c39-47c9-9884-49deb2c91857","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142826289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129322024-02-13T21:44:32.413Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142857698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
129332024-02-13T21:44:32.413Z{"msg":"c92ba656-1c39-47c9-9884-49deb2c91857 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142950434Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129342024-02-13T21:44:32.413Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142978484Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129352024-02-13T21:44:32.413Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.142994328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129362024-02-13T21:44:32.413Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143026317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129372024-02-13T21:44:32.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.14304553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129382024-02-13T21:44:32.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143069511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129392024-02-13T21:44:32.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143086205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129402024-02-13T21:44:32.414Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143115874Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129412024-02-13T21:44:32.414Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143137616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129422024-02-13T21:44:32.414Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143191876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129432024-02-13T21:44:32.414Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143246397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129442024-02-13T21:44:32.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143345481Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129452024-02-13T21:44:32.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.143372511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"io task","client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129462024-02-13T21:44:32.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144155277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129472024-02-13T21:44:32.415Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144221873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129482024-02-13T21:44:32.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144267476Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129492024-02-13T21:44:32.415Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144306602Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129502024-02-13T21:44:32.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144776452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129512024-02-13T21:44:32.416Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144818536Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129522024-02-13T21:44:32.416Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144840808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129532024-02-13T21:44:32.416Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.144855853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129542024-02-13T21:44:32.416Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145355252Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129552024-02-13T21:44:32.416Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145391908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129562024-02-13T21:44:32.416Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145407323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129572024-02-13T21:44:32.416Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145509566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129582024-02-13T21:44:32.416Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145546572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129592024-02-13T21:44:32.416Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145618256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129602024-02-13T21:44:32.417Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145636919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129612024-02-13T21:44:32.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.145983804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129622024-02-13T21:44:32.417Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146025938Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129632024-02-13T21:44:32.422Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 220, 220, 222, 204, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146042243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129642024-02-13T21:44:32.423Z{"msg":"[0]R generation[0..12]: [104, 114, 118, 118, 119, 110, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146065504Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129652024-02-13T21:44:32.423Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146080729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129662024-02-13T21:44:32.423Z{"msg":"[1]R flush_numbers[0..12]: [193, 210, 220, 220, 222, 204, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146096373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129672024-02-13T21:44:32.423Z{"msg":"[1]R generation[0..12]: [104, 114, 118, 118, 119, 110, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146110958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129682024-02-13T21:44:32.423Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.14613213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129692024-02-13T21:44:32.423Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 220, 220, 222, 204, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146147224Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129702024-02-13T21:44:32.423Z{"msg":"[2]R generation[0..12]: [104, 114, 118, 118, 119, 110, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
129712024-02-13T21:44:32.423Z,"time":"2024-02-13T21:44:31.146167867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129722024-02-13T21:44:32.423Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.14618673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129732024-02-13T21:44:32.423Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146206653Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129742024-02-13T21:44:32.424Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146220268Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129752024-02-13T21:44:32.424Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146247798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129762024-02-13T21:44:32.424Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146279906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"":"downstairs","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129772024-02-13T21:44:32.424Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.14629833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129782024-02-13T21:44:32.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146330208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"0","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129792024-02-13T21:44:32.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146361197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"1","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129802024-02-13T21:44:32.424Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146394155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"client":"2","":"downstairs","session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129812024-02-13T21:44:32.424Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146424834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129822024-02-13T21:44:32.425Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146448285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129832024-02-13T21:44:32.425Z{"msg":"c92ba656-1c39-47c9-9884-49deb2c91857 is now active with session: 36bba27b-ea4c-4d0e-b770-db1745ce7106","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129842024-02-13T21:44:32.425Z,"time":"2024-02-13T21:44:31.146469847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129852024-02-13T21:44:32.425Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146496588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618,"session_id":"36bba27b-ea4c-4d0e-b770-db1745ce7106"}
129862024-02-13T21:44:32.425Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:31.146510773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
129872024-02-13T21:44:32.425ZLoading write count information from file "/tmp/test_repair_verify.data"
129882024-02-13T21:44:32.425ZRun Repair workload
129892024-02-13T21:44:32.425Z01/30 Write block 301 len 4096 data: 7
129902024-02-13T21:44:32.425Z02/30 Write block 362 len 32768 data: 9 11 9 8 7 11 10 10
129912024-02-13T21:44:32.426Z03/30 Read block 592 len 16384
129922024-02-13T21:44:32.426Z04/30 Read block 420 len 40960
129932024-02-13T21:44:32.426Z05/30 Write block 50 len 36864 data: 8 8 8 7 9 10 11 12 11
129942024-02-13T21:44:32.426Z06/30 Write block 372 len 28672 data: 14 13 11 13 12 10 11
129952024-02-13T21:44:32.426Z07/30 Flush
129962024-02-13T21:44:32.426Z08/30 Write block 535 len 36864 data: 8 6 7 8 11 14 14 11 11
129972024-02-13T21:44:32.426Z09/30 Read block 487 len 36864
129982024-02-13T21:44:32.430Z10/30 Write block 374 len 16384 data: 12 14 13 11
129992024-02-13T21:44:32.431Z11/30 Read block 261 len 32768
130002024-02-13T21:44:32.431Z12/30 Read block 165 len 20480
130012024-02-13T21:44:32.431Z13/30 Flush
130022024-02-13T21:44:32.431Z14/30 Write block 268 len 16384 data: 12 11 11 11
130032024-02-13T21:44:32.431Z15/30 Write block 249 len 12288 data: 9 10 12
130042024-02-13T21:44:32.431Z16/30 Write block 319 len 40960 data: 15 14 13 14 16 15 16 16 14 13
130052024-02-13T21:44:32.432Z17/30 Read block 462 len 32768
130062024-02-13T21:44:32.432Z18/30 Flush
130072024-02-13T21:44:32.436Z19/30 Write block 280 len 32768 data: 11 8 6 6 7 8 7 7
130082024-02-13T21:44:32.436Z20/30 Read block 480 len 40960
130092024-02-13T21:44:32.436Z21/30 Flush
130102024-02-13T21:44:32.440Z22/30 Write block 593 len 8192 data: 5 6
130112024-02-13T21:44:32.440Z23/30 Read block 21 len 28672
130122024-02-13T21:44:32.440Z24/30 Write block 115 len 12288 data: 8 10 11
130132024-02-13T21:44:32.440Z25/30 Write block 453 len 24576 data: 9 9 11 12 12 12
130142024-02-13T21:44:32.441Z26/30 Write block 403 len 20480 data: 8 8 8 6 6
130152024-02-13T21:44:32.444Z27/30 Read block 120 len 40960
130162024-02-13T21:44:32.445Z28/30 Read block 494 len 36864
130172024-02-13T21:44:32.445Z29/30 Read block 437 len 40960
130182024-02-13T21:44:32.445Z30/30 Write block 273 len 20480 data: 11 12 13 13 14
130192024-02-13T21:44:32.445Z----------------------------------------------------------------
130202024-02-13T21:44:32.445Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
130212024-02-13T21:44:32.445ZStates: Active Active Active
130222024-02-13T21:44:32.445ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130232024-02-13T21:44:32.445Z 1 Acked 1000 Write 1 Done Done Sent false
130242024-02-13T21:44:32.445Z 2 Acked 1001 Write 8 Done Done Sent false
130252024-02-13T21:44:32.445Z 3 Acked 1002 Read 4 Done Done Sent false
130262024-02-13T21:44:32.445Z 4 Acked 1003 Read 10 Done Done Sent false
130272024-02-13T21:44:32.445Z 5 Acked 1004 Write 9 Done Done Sent false
130282024-02-13T21:44:32.445Z 6 Acked 1005 Write 7 Done Done Sent false
130292024-02-13T21:44:32.445Z 7 Acked 1006 Flush 0 Done Done Sent false
130302024-02-13T21:44:32.446Z 8 Acked 1007 Write 9 Done Done Sent false
130312024-02-13T21:44:32.446Z 9 Acked 1008 Read 9 Done Done Sent false
130322024-02-13T21:44:32.446Z 10 Acked 1009 Write 4 Done Done Sent false
130332024-02-13T21:44:32.446Z 11 Acked 1010 Read 8 Done Done Sent false
130342024-02-13T21:44:32.446Z 12 Acked 1011 Read 5 Done Done Sent false
130352024-02-13T21:44:32.446Z 13 Acked 1012 Flush 0 Done Done Sent false
130362024-02-13T21:44:32.446Z 14 Acked 1013 Write 4 Done Done Sent false
130372024-02-13T21:44:32.446Z 15 Acked 1014 Write 3 Done Done Sent false
130382024-02-13T21:44:32.446Z 16 Acked 1015 Write 10 Done Done Sent false
130392024-02-13T21:44:32.446Z 17 Acked 1016 Read 8 Done Done Sent false
130402024-02-13T21:44:32.446Z 18 Acked 1017 Flush 0 Done Done Sent false
130412024-02-13T21:44:32.446Z 19 Acked 1018 Write 8 Done Done Sent false
130422024-02-13T21:44:32.446Z 20 Acked 1019 Read 10 Done Done Sent false
130432024-02-13T21:44:32.446Z 21 Acked 1020 Flush 0 Done Done Sent false
130442024-02-13T21:44:32.446Z 22 Acked 1021 Write 2 Done Done Sent false
130452024-02-13T21:44:32.447Z 23 Acked 1022 Read 7 Done Done Sent false
130462024-02-13T21:44:32.447Z 24 Acked 1023 Write 3 Done Done Sent false
130472024-02-13T21:44:32.447Z 25 Acked 1024 Write 6 Done Done Sent false
130482024-02-13T21:44:32.447Z 26 Acked 1025 Write 5 Done Done Sent false
130492024-02-13T21:44:32.447Z 27 Acked 1026 Read 10 Done Done Sent false
130502024-02-13T21:44:32.447Z 28 Acked 1027 Read 9 Done Done Sent false
130512024-02-13T21:44:32.447Z 29 Acked 1028 Read 10 Done Done Sent false
130522024-02-13T21:44:32.447Z 30 Acked 1029 Write 5 Sent Sent Sent false
130532024-02-13T21:44:32.447Z STATES DS:0 DS:1 DS:2 TOTAL
130542024-02-13T21:44:32.447Z New 0 0 0 0
130552024-02-13T21:44:32.447Z Sent 1 1 30 32
130562024-02-13T21:44:32.447Z Done 29 29 0 58
130572024-02-13T21:44:32.447Z Skipped 0 0 0 0
130582024-02-13T21:44:32.447Z Error 0 0 0 0
130592024-02-13T21:44:32.447ZLast Flush: 1020 1020 0
130602024-02-13T21:44:32.447ZDownstairs last five completed:
130612024-02-13T21:44:32.447ZUpstairs last five completed: 30 29 28 27 26
130622024-02-13T21:44:32.447Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:31.180488568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1618}
130632024-02-13T21:44:32.448ZWrote out file "/tmp/test_repair_verify.data"
130642024-02-13T21:44:32.451Z
130652024-02-13T21:44:32.451ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130662024-02-13T21:44:32.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130672024-02-13T21:44:32.468Z 0 000-019 104 104 104 193 193 193 F F F
130682024-02-13T21:44:32.469Z 1 020-039 114 114 114 210 210 210 F F F
130692024-02-13T21:44:32.469Z 2 040-059 120 120 118 223 223 220 F F F <---
130702024-02-13T21:44:32.469Z 3 060-079 118 118 118 220 220 220 F F F
130712024-02-13T21:44:32.469Z 4 080-099 119 119 119 222 222 222 F F F
130722024-02-13T21:44:32.469Z 5 100-119 110 110 110 204 204 204 T T F <---
130732024-02-13T21:44:32.469Z 6 120-139 118 118 118 220 220 220 F F F
130742024-02-13T21:44:32.470Z 7 140-159 118 118 118 220 220 220 F F F
130752024-02-13T21:44:32.470Z 8 160-179 118 118 118 220 220 220 F F F
130762024-02-13T21:44:32.470Z 9 180-199 110 110 110 204 204 204 F F F
130772024-02-13T21:44:32.487Z 10 200-219 115 115 115 211 211 211 F F F
130782024-02-13T21:44:32.488Z 11 220-239 110 110 110 203 203 203 F F F
130792024-02-13T21:44:32.488Z 12 240-259 120 120 118 225 225 220 F F F <---
130802024-02-13T21:44:32.488Z 13 260-279 120 120 116 225 225 212 T T F <---
130812024-02-13T21:44:32.488Z 14 280-299 120 120 116 226 226 212 F F F <---
130822024-02-13T21:44:32.488Z 15 300-319 120 120 119 225 225 222 F F F <---
130832024-02-13T21:44:32.488Z 16 320-339 120 120 118 225 225 220 F F F <---
130842024-02-13T21:44:32.488Z 17 340-359 117 117 117 215 215 215 F F F
130852024-02-13T21:44:32.488Z 18 360-379 120 120 116 224 224 212 F F F <---
130862024-02-13T21:44:32.488Z 19 380-399 118 118 118 220 220 220 F F F
130872024-02-13T21:44:32.489Z 20 400-419 118 118 118 220 220 220 T T F <---
130882024-02-13T21:44:32.489Z 21 420-439 118 118 118 220 220 220 F F F
130892024-02-13T21:44:32.489Z 22 440-459 118 118 118 220 220 220 T T F <---
130902024-02-13T21:44:32.489Z 23 460-479 115 115 115 211 211 211 F F F
130912024-02-13T21:44:32.489Z 24 480-499 113 113 113 208 208 208 F F F
130922024-02-13T21:44:32.489Z 25 500-519 115 115 115 211 211 211 F F F
130932024-02-13T21:44:32.489Z 26 520-539 120 120 118 224 224 221 F F F <---
130942024-02-13T21:44:32.489Z 27 540-559 120 120 116 224 224 212 F F F <---
130952024-02-13T21:44:32.489Z 28 560-579 115 115 115 211 211 211 F F F
130962024-02-13T21:44:32.490Z 29 580-599 115 115 115 211 211 211 T T F <---
130972024-02-13T21:44:32.490ZMax gen: 120, Max flush: 226
130982024-02-13T21:44:32.490ZError: Difference in extent metadata found!
130992024-02-13T21:44:32.490ZA Difference in extent metadata is expected here
131002024-02-13T21:44:32.490ZOn loop 60
131012024-02-13T21:44:32.490Z
131022024-02-13T21:44:32.490ZVerifying data now
131032024-02-13T21:44:37.104ZNo change after verify
131042024-02-13T21:44:37.104ZLoop: 60 Downstairs dump after verify (and repair):
131052024-02-13T21:44:37.118ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131062024-02-13T21:44:37.118Z 0 000-019 104 104 104 193 193 193 F F F
131072024-02-13T21:44:37.118Z 1 020-039 114 114 114 210 210 210 F F F
131082024-02-13T21:44:37.118Z 2 040-059 120 120 120 223 223 223 F F F
131092024-02-13T21:44:37.119Z 3 060-079 118 118 118 220 220 220 F F F
131102024-02-13T21:44:37.119Z 4 080-099 119 119 119 222 222 222 F F F
131112024-02-13T21:44:37.119Z 5 100-119 121 121 121 227 227 227 F F F
131122024-02-13T21:44:37.119Z 6 120-139 118 118 118 220 220 220 F F F
131132024-02-13T21:44:37.119Z 7 140-159 118 118 118 220 220 220 F F F
131142024-02-13T21:44:37.119Z 8 160-179 118 118 118 220 220 220 F F F
131152024-02-13T21:44:37.119Z 9 180-199 110 110 110 204 204 204 F F F
131162024-02-13T21:44:37.122Z 10 200-219 115 115 115 211 211 211 F F F
131172024-02-13T21:44:37.122Z 11 220-239 110 110 110 203 203 203 F F F
131182024-02-13T21:44:37.122Z 12 240-259 120 120 120 225 225 225 F F F
131192024-02-13T21:44:37.122Z 13 260-279 121 121 121 227 227 227 F F F
131202024-02-13T21:44:37.122Z 14 280-299 120 120 120 226 226 226 F F F
131212024-02-13T21:44:37.122Z 15 300-319 120 120 120 225 225 225 F F F
131222024-02-13T21:44:37.122Z 16 320-339 120 120 120 225 225 225 F F F
131232024-02-13T21:44:37.122Z 17 340-359 117 117 117 215 215 215 F F F
131242024-02-13T21:44:37.122Z 18 360-379 120 120 120 224 224 224 F F F
131252024-02-13T21:44:37.122Z 19 380-399 118 118 118 220 220 220 F F F
131262024-02-13T21:44:37.122Z 20 400-419 121 121 121 227 227 227 F F F
131272024-02-13T21:44:37.122Z 21 420-439 118 118 118 220 220 220 F F F
131282024-02-13T21:44:37.122Z 22 440-459 121 121 121 227 227 227 F F F
131292024-02-13T21:44:37.122Z 23 460-479 115 115 115 211 211 211 F F F
131302024-02-13T21:44:37.122Z 24 480-499 113 113 113 208 208 208 F F F
131312024-02-13T21:44:37.122Z 25 500-519 115 115 115 211 211 211 F F F
131322024-02-13T21:44:37.122Z 26 520-539 120 120 120 224 224 224 F F F
131332024-02-13T21:44:37.122Z 27 540-559 120 120 120 224 224 224 F F F
131342024-02-13T21:44:37.123Z 28 560-579 115 115 115 211 211 211 F F F
131352024-02-13T21:44:37.123Z 29 580-599 121 121 121 227 227 227 F F F
131362024-02-13T21:44:37.123ZMax gen: 121, Max flush: 227
131372024-02-13T21:44:37.123Z
131382024-02-13T21:44:37.123ZBegin loop 61
131392024-02-13T21:44:37.123ZDownstairs to restart: 2
131402024-02-13T21:44:37.139Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873348337Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
131412024-02-13T21:44:37.139Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.87358832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
131422024-02-13T21:44:37.144Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873623397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
131432024-02-13T21:44:37.144Z{"msg":"Crucible 4285f7c6-5ea1-456f-81f8-429076854b20 has session id: 8a553e4a-ede3-40a7-8107-9ef2c8a087d4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873647788Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131442024-02-13T21:44:37.144Z{"msg":"Crucible runtime is spawned
131452024-02-13T21:44:37.144ZUpstairs opts: Upstairs UUID: 4285f7c6-5ea1-456f-81f8-429076854b20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873701709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131462024-02-13T21:44:37.144Z{"msg":"Crucible stats registered with UUID: 4285f7c6-5ea1-456f-81f8-429076854b20","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873740894Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131472024-02-13T21:44:37.144Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873766055Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
131482024-02-13T21:44:37.144Z{"msg":"4285f7c6-5ea1-456f-81f8-429076854b20 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873895438Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131492024-02-13T21:44:37.144Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873925137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131502024-02-13T21:44:37.144Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873940592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131512024-02-13T21:44:37.144Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873961284Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131522024-02-13T21:44:37.144Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.873975369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131532024-02-13T21:44:37.144Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874003229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131542024-02-13T21:44:37.144Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874017984Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131552024-02-13T21:44:37.144Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874039426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131562024-02-13T21:44:37.144Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874072404Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131572024-02-13T21:44:37.145Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874107901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131582024-02-13T21:44:37.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874311167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131592024-02-13T21:44:37.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874364538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131602024-02-13T21:44:37.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.874393248Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"io task","client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131612024-02-13T21:44:37.145Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875105229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131622024-02-13T21:44:37.145Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875140037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131632024-02-13T21:44:37.145Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875155171Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131642024-02-13T21:44:37.145Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875175354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131652024-02-13T21:44:37.145Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875697085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131662024-02-13T21:44:37.145Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.875836544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131672024-02-13T21:44:37.145Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876138245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131682024-02-13T21:44:37.145Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876173442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131692024-02-13T21:44:37.145Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876189116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131702024-02-13T21:44:37.145Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876208799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131712024-02-13T21:44:37.145Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876223054Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131722024-02-13T21:44:37.145Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.87634457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131732024-02-13T21:44:37.145Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.87637131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131742024-02-13T21:44:37.145Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876385975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131752024-02-13T21:44:37.145Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.876776843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131762024-02-13T21:44:37.150Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877242324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131772024-02-13T21:44:37.150Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877287748Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131782024-02-13T21:44:37.150Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 227, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877350535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131792024-02-13T21:44:37.150Z{"msg":"[0]R generation[0..12]: [104, 114, 120, 118, 119, 121, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877389241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131802024-02-13T21:44:37.150Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877410193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131812024-02-13T21:44:37.150Z{"msg":"[1]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 227, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
131822024-02-13T21:44:37.150Z,"time":"2024-02-13T21:44:35.877429336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131832024-02-13T21:44:37.150Z{"msg":"[1]R generation[0..12]: [104, 114, 120, 118, 119, 121, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877456886Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131842024-02-13T21:44:37.150Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877476419Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131852024-02-13T21:44:37.150Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 227, 220, 220, 220, 204, 211, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877494333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131862024-02-13T21:44:37.150Z{"msg":"[2]R generation[0..12]: [104, 114, 120, 118, 119, 121, 118, 118, 118, 110, 115, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877521243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131872024-02-13T21:44:37.150Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877539586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131882024-02-13T21:44:37.150Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877564997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131892024-02-13T21:44:37.150Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877582601Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131902024-02-13T21:44:37.150Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877607092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131912024-02-13T21:44:37.150Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877623896Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"":"downstairs","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131922024-02-13T21:44:37.150Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877648957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131932024-02-13T21:44:37.150Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877664621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"0","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131942024-02-13T21:44:37.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877689352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"1","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131952024-02-13T21:44:37.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877705996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"client":"2","":"downstairs","session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131962024-02-13T21:44:37.151Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877730507Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131972024-02-13T21:44:37.151Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877746082Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131982024-02-13T21:44:37.151Z{"msg":"4285f7c6-5ea1-456f-81f8-429076854b20 is now active with session: 8a553e4a-ede3-40a7-8107-9ef2c8a087d4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877762925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
131992024-02-13T21:44:37.151Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877787336Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626,"session_id":"8a553e4a-ede3-40a7-8107-9ef2c8a087d4"}
132002024-02-13T21:44:37.151Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:35.877811768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
132012024-02-13T21:44:37.151ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132022024-02-13T21:44:37.151ZLoading write count information from file "/tmp/test_repair_verify.data"
132032024-02-13T21:44:37.151ZRun Repair workload
132042024-02-13T21:44:37.151Z01/30 Write block 176 len 40960 data: 6 9 9 8 9 8 7 10 11 13
132052024-02-13T21:44:37.151Z02/30 Read block 51 len 28672
132062024-02-13T21:44:37.151Z03/30 Write block 159 len 20480 data: 9 8 9 9 8
132072024-02-13T21:44:37.151Z04/30 Write block 384 len 24576 data: 10 10 11 10 10 9
132082024-02-13T21:44:37.154Z05/30 Write block 110 len 36864 data: 5 8 8 8 8 9 11 12 9
132092024-02-13T21:44:37.154Z06/30 Write block 417 len 12288 data: 17 15 14
132102024-02-13T21:44:37.154Z07/30 Write block 203 len 40960 data: 8 6 6 3 4 6 7 8 8 8
132112024-02-13T21:44:37.154Z08/30 Read block 275 len 36864
132122024-02-13T21:44:37.155Z09/30 Write block 263 len 4096 data: 7
132132024-02-13T21:44:37.155Z10/30 Read block 516 len 32768
132142024-02-13T21:44:37.155Z11/30 Read block 25 len 8192
132152024-02-13T21:44:37.155Z12/30 Read block 286 len 24576
132162024-02-13T21:44:37.158Z13/30 Read block 232 len 12288
132172024-02-13T21:44:37.158Z14/30 Write block 124 len 28672 data: 9 14 14 13 10 10 7
132182024-02-13T21:44:37.158Z15/30 Write block 366 len 36864 data: 8 12 11 11 10 12 15 14 13
132192024-02-13T21:44:37.161Z16/30 Write block 213 len 36864 data: 9 9 10 12 13 14 15 14 13
132202024-02-13T21:44:37.161Z17/30 Write block 490 len 16384 data: 7 6 6 9
132212024-02-13T21:44:37.161Z18/30 Write block 315 len 4096 data: 10
132222024-02-13T21:44:37.161Z19/30 Read block 284 len 24576
132232024-02-13T21:44:37.161Z20/30 Write block 414 len 20480 data: 11 12 16 18 16
132242024-02-13T21:44:37.164Z21/30 Read block 49 len 36864
132252024-02-13T21:44:37.164Z22/30 Flush
132262024-02-13T21:44:37.170Z23/30 Write block 473 len 8192 data: 15 15
132272024-02-13T21:44:37.173Z24/30 Write block 548 len 20480 data: 7 11 11 15 12
132282024-02-13T21:44:37.173Z25/30 Write block 406 len 24576 data: 7 7 8 11 10 13
132292024-02-13T21:44:37.173Z26/30 Write block 318 len 28672 data: 13 16 15 14 15 17 16
132302024-02-13T21:44:37.173Z27/30 Read block 349 len 40960
132312024-02-13T21:44:37.173Z28/30 Write block 490 len 28672 data: 8 7 7 10 11 9 9
132322024-02-13T21:44:37.174Z29/30 Read block 243 len 4096
132332024-02-13T21:44:37.174Z30/30 Write block 137 len 32768 data: 8 12 10 11 12 16 13 14
132342024-02-13T21:44:37.177Z----------------------------------------------------------------
132352024-02-13T21:44:37.177Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
132362024-02-13T21:44:37.177ZStates: Active Active Active
132372024-02-13T21:44:37.177ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132382024-02-13T21:44:37.177Z 1 Acked 1000 Write 10 Done Done Sent false
132392024-02-13T21:44:37.177Z 2 Acked 1001 Read 7 Done Done Sent false
132402024-02-13T21:44:37.177Z 3 Acked 1002 Write 5 Done Done Sent false
132412024-02-13T21:44:37.177Z 4 Acked 1003 Write 6 Done Done Sent false
132422024-02-13T21:44:37.178Z 5 Acked 1004 Write 9 Done Done Sent false
132432024-02-13T21:44:37.178Z 6 Acked 1005 Write 3 Done Done Sent false
132442024-02-13T21:44:37.178Z 7 Acked 1006 Write 10 Done Done Sent false
132452024-02-13T21:44:37.178Z 8 Acked 1007 Read 9 Done Done Sent false
132462024-02-13T21:44:37.178Z 9 Acked 1008 Write 1 Done Done Sent false
132472024-02-13T21:44:37.178Z 10 Acked 1009 Read 8 Done Done Sent false
132482024-02-13T21:44:37.178Z 11 Acked 1010 Read 2 Done Done Sent false
132492024-02-13T21:44:37.178Z 12 Acked 1011 Read 6 Done Done Sent false
132502024-02-13T21:44:37.178Z 13 Acked 1012 Read 3 Done Done Sent false
132512024-02-13T21:44:37.178Z 14 Acked 1013 Write 7 Done Done Sent false
132522024-02-13T21:44:37.178Z 15 Acked 1014 Write 9 Done Done Sent false
132532024-02-13T21:44:37.178Z 16 Acked 1015 Write 9 Done Done Sent false
132542024-02-13T21:44:37.178Z 17 Acked 1016 Write 4 Done Done Sent false
132552024-02-13T21:44:37.178Z 18 Acked 1017 Write 1 Done Done Sent false
132562024-02-13T21:44:37.178Z 19 Acked 1018 Read 6 Done Done Sent false
132572024-02-13T21:44:37.178Z 20 Acked 1019 Write 5 Done Done Sent false
132582024-02-13T21:44:37.178Z 21 Acked 1020 Read 9 Done Done Sent false
132592024-02-13T21:44:37.178Z 22 Acked 1021 Flush 0 Done Done Sent false
132602024-02-13T21:44:37.178Z 23 Acked 1022 Write 2 Done Done Sent false
132612024-02-13T21:44:37.178Z 24 Acked 1023 Write 5 Done Done Sent false
132622024-02-13T21:44:37.178Z 25 Acked 1024 Write 6 Done Done Sent false
132632024-02-13T21:44:37.178Z 26 Acked 1025 Write 7 Done Done Sent false
132642024-02-13T21:44:37.178Z 27 Acked 1026 Read 10 Done Done Sent false
132652024-02-13T21:44:37.178Z 28 Acked 1027 Write 7 Done Done Sent false
132662024-02-13T21:44:37.178Z 29 Acked 1028 Read 1 Done Done Sent false
132672024-02-13T21:44:37.179Z 30 Acked 1029 Write 8 Sent Sent Sent false
132682024-02-13T21:44:37.179Z STATES DS:0 DS:1 DS:2 TOTAL
132692024-02-13T21:44:37.179Z New 0 0 0 0
132702024-02-13T21:44:37.179Z Sent 1 1 30 32
132712024-02-13T21:44:37.179Z Done 29 29 0 58
132722024-02-13T21:44:37.179Z Skipped 0 0 0 0
132732024-02-13T21:44:37.179Z Error 0 0 0 0
132742024-02-13T21:44:37.179ZLast Flush: 1021 1021 0
132752024-02-13T21:44:37.179ZDownstairs last five completed:
132762024-02-13T21:44:37.179ZUpstairs last five completed: 30 29 28 27 26
132772024-02-13T21:44:37.179Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:35.908560364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1626}
132782024-02-13T21:44:37.179ZWrote out file "/tmp/test_repair_verify.data"
132792024-02-13T21:44:37.182Z
132802024-02-13T21:44:37.182ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132812024-02-13T21:44:37.198ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132822024-02-13T21:44:37.198Z 0 000-019 104 104 104 193 193 193 F F F
132832024-02-13T21:44:37.198Z 1 020-039 114 114 114 210 210 210 F F F
132842024-02-13T21:44:37.198Z 2 040-059 120 120 120 223 223 223 F F F
132852024-02-13T21:44:37.198Z 3 060-079 118 118 118 220 220 220 F F F
132862024-02-13T21:44:37.198Z 4 080-099 119 119 119 222 222 222 F F F
132872024-02-13T21:44:37.198Z 5 100-119 122 122 121 228 228 227 F F F <---
132882024-02-13T21:44:37.201Z 6 120-139 122 122 118 228 228 220 T T F <---
132892024-02-13T21:44:37.201Z 7 140-159 122 122 118 228 228 220 T T F <---
132902024-02-13T21:44:37.202Z 8 160-179 122 122 118 228 228 220 F F F <---
132912024-02-13T21:44:37.202Z 9 180-199 122 122 110 228 228 204 F F F <---
132922024-02-13T21:44:37.202Z 10 200-219 122 122 115 228 228 211 F F F <---
132932024-02-13T21:44:37.202Z 11 220-239 122 122 110 228 228 203 F F F <---
132942024-02-13T21:44:37.202Z 12 240-259 120 120 120 225 225 225 F F F
132952024-02-13T21:44:37.202Z 13 260-279 122 122 121 228 228 227 F F F <---
132962024-02-13T21:44:37.202Z 14 280-299 120 120 120 226 226 226 F F F
132972024-02-13T21:44:37.202Z 15 300-319 122 122 120 228 228 225 T T F <---
132982024-02-13T21:44:37.202Z 16 320-339 120 120 120 225 225 225 T T F <---
132992024-02-13T21:44:37.202Z 17 340-359 117 117 117 215 215 215 F F F
133002024-02-13T21:44:37.202Z 18 360-379 122 122 120 228 228 224 F F F <---
133012024-02-13T21:44:37.202Z 19 380-399 122 122 118 228 228 220 F F F <---
133022024-02-13T21:44:37.202Z 20 400-419 122 122 121 228 228 227 T T F <---
133032024-02-13T21:44:37.202Z 21 420-439 118 118 118 220 220 220 F F F
133042024-02-13T21:44:37.202Z 22 440-459 121 121 121 227 227 227 F F F
133052024-02-13T21:44:37.202Z 23 460-479 115 115 115 211 211 211 T T F <---
133062024-02-13T21:44:37.202Z 24 480-499 122 122 113 228 228 208 T T F <---
133072024-02-13T21:44:37.202Z 25 500-519 115 115 115 211 211 211 F F F
133082024-02-13T21:44:37.202Z 26 520-539 120 120 120 224 224 224 F F F
133092024-02-13T21:44:37.202Z 27 540-559 120 120 120 224 224 224 T T F <---
133102024-02-13T21:44:37.202Z 28 560-579 115 115 115 211 211 211 F F F
133112024-02-13T21:44:37.202Z 29 580-599 121 121 121 227 227 227 F F F
133122024-02-13T21:44:37.202ZMax gen: 122, Max flush: 228
133132024-02-13T21:44:37.202ZError: Difference in extent metadata found!
133142024-02-13T21:44:37.203ZA Difference in extent metadata is expected here
133152024-02-13T21:44:37.203ZOn loop 61
133162024-02-13T21:44:37.203Z
133172024-02-13T21:44:37.206ZVerifying data now
133182024-02-13T21:44:41.979ZNo change after verify
133192024-02-13T21:44:41.979ZLoop: 61 Downstairs dump after verify (and repair):
133202024-02-13T21:44:41.994ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133212024-02-13T21:44:41.994Z 0 000-019 104 104 104 193 193 193 F F F
133222024-02-13T21:44:41.994Z 1 020-039 114 114 114 210 210 210 F F F
133232024-02-13T21:44:41.994Z 2 040-059 120 120 120 223 223 223 F F F
133242024-02-13T21:44:41.994Z 3 060-079 118 118 118 220 220 220 F F F
133252024-02-13T21:44:41.995Z 4 080-099 119 119 119 222 222 222 F F F
133262024-02-13T21:44:41.998Z 5 100-119 122 122 122 228 228 228 F F F
133272024-02-13T21:44:41.999Z 6 120-139 123 123 123 229 229 229 F F F
133282024-02-13T21:44:41.999Z 7 140-159 123 123 123 229 229 229 F F F
133292024-02-13T21:44:41.999Z 8 160-179 122 122 122 228 228 228 F F F
133302024-02-13T21:44:41.999Z 9 180-199 122 122 122 228 228 228 F F F
133312024-02-13T21:44:41.999Z 10 200-219 122 122 122 228 228 228 F F F
133322024-02-13T21:44:41.999Z 11 220-239 122 122 122 228 228 228 F F F
133332024-02-13T21:44:41.999Z 12 240-259 120 120 120 225 225 225 F F F
133342024-02-13T21:44:41.999Z 13 260-279 122 122 122 228 228 228 F F F
133352024-02-13T21:44:41.999Z 14 280-299 120 120 120 226 226 226 F F F
133362024-02-13T21:44:41.999Z 15 300-319 123 123 123 229 229 229 F F F
133372024-02-13T21:44:42.000Z 16 320-339 123 123 123 229 229 229 F F F
133382024-02-13T21:44:42.000Z 17 340-359 117 117 117 215 215 215 F F F
133392024-02-13T21:44:42.000Z 18 360-379 122 122 122 228 228 228 F F F
133402024-02-13T21:44:42.000Z 19 380-399 122 122 122 228 228 228 F F F
133412024-02-13T21:44:42.000Z 20 400-419 123 123 123 229 229 229 F F F
133422024-02-13T21:44:42.000Z 21 420-439 118 118 118 220 220 220 F F F
133432024-02-13T21:44:42.002Z 22 440-459 121 121 121 227 227 227 F F F
133442024-02-13T21:44:42.002Z 23 460-479 123 123 123 229 229 229 F F F
133452024-02-13T21:44:42.002Z 24 480-499 123 123 123 229 229 229 F F F
133462024-02-13T21:44:42.002Z 25 500-519 115 115 115 211 211 211 F F F
133472024-02-13T21:44:42.002Z 26 520-539 120 120 120 224 224 224 F F F
133482024-02-13T21:44:42.002Z 27 540-559 123 123 123 229 229 229 F F F
133492024-02-13T21:44:42.002Z 28 560-579 115 115 115 211 211 211 F F F
133502024-02-13T21:44:42.002Z 29 580-599 121 121 121 227 227 227 F F F
133512024-02-13T21:44:42.003ZMax gen: 123, Max flush: 229
133522024-02-13T21:44:42.003Z
133532024-02-13T21:44:42.003ZBegin loop 62
133542024-02-13T21:44:42.003ZDownstairs to restart: 1
133552024-02-13T21:44:42.015Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.747422814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
133562024-02-13T21:44:42.015Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time"Crucible runtime is spawned
133572024-02-13T21:44:42.027Z:"2024-02-13T21:44:40.747784933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
133582024-02-13T21:44:42.027Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.747873871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
133592024-02-13T21:44:42.027Z{"msg":"Crucible 742233e3-c3a0-4f36-ab38-8cf75ce20c15 has session id: 4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.747920394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133602024-02-13T21:44:42.027Z{"msg":"Upstairs opts: Upstairs UUID: 742233e3-c3a0-4f36-ab38-8cf75ce20c15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.747961399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133612024-02-13T21:44:42.028Z{"msg":"Crucible stats registered with UUID: 742233e3-c3a0-4f36-ab38-8cf75ce20c15","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.747987679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133622024-02-13T21:44:42.028Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748002554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
133632024-02-13T21:44:42.028Z{"msg":"742233e3-c3a0-4f36-ab38-8cf75ce20c15 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748122241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133642024-02-13T21:44:42.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748147461Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133652024-02-13T21:44:42.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748163646Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133662024-02-13T21:44:42.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748190166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133672024-02-13T21:44:42.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748204281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133682024-02-13T21:44:42.028Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748220145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133692024-02-13T21:44:42.028Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748253893Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133702024-02-13T21:44:42.028Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748279173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133712024-02-13T21:44:42.028Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748312781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133722024-02-13T21:44:42.028Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748335603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133732024-02-13T21:44:42.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748450022Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133742024-02-13T21:44:42.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748496485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133752024-02-13T21:44:42.029Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.748527454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"io task","client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133762024-02-13T21:44:42.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.749209556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133772024-02-13T21:44:42.029Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.74925298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133782024-02-13T21:44:42.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.749292456Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133792024-02-13T21:44:42.029Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.749323355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133802024-02-13T21:44:42.029Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.749819995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133812024-02-13T21:44:42.029Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.749866368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133822024-02-13T21:44:42.029Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750157392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133832024-02-13T21:44:42.029Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750248919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133842024-02-13T21:44:42.029Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.75041083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133852024-02-13T21:44:42.029Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750452675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133862024-02-13T21:44:42.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750491381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133872024-02-13T21:44:42.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750515642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133882024-02-13T21:44:42.030Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750538684Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133892024-02-13T21:44:42.030Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750553109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133902024-02-13T21:44:42.030Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.750732374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133912024-02-13T21:44:42.030Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751125941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133922024-02-13T21:44:42.030Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751166016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133932024-02-13T21:44:42.030Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 228, 229, 229, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.7511823Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133942024-02-13T21:44:42.030Z{"msg":"[0]R generation[0..12]: [104, 114, 120, 118, 119, 122, 123, 123, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751197335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133952024-02-13T21:44:42.030Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751218797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133962024-02-13T21:44:42.030Z{"msg":"[1]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 228, 229, 229, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751233472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133972024-02-13T21:44:42.030Z{"msg":"[1]R generation[0..12]: [104, 114, 120, 118, 119, 122, 123, 123, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751254364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133982024-02-13T21:44:42.030Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751268979Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
133992024-02-13T21:44:42.031Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 223, 220, 222, 228, 229, 229, 228, 228, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751299358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134002024-02-13T21:44:42.031ZWait for a query_work_queue command to finish before sending IO
134012024-02-13T21:44:42.031Z{"msg":"[2]R generation[0..12]: [104, 114, 120, 118, 119, 122, 123, 123, 122, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751327118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134022024-02-13T21:44:42.031Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751358687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134032024-02-13T21:44:42.031Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751382208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134042024-02-13T21:44:42.031Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751409348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134052024-02-13T21:44:42.031Z{"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.75143231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134062024-02-13T21:44:42.031Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.75145923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"":"downstairs","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134072024-02-13T21:44:42.037Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751483081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134082024-02-13T21:44:42.037Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751509142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"0","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134092024-02-13T21:44:42.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751532624Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"1","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134102024-02-13T21:44:42.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751559863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"client":"2","":"downstairs","session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134112024-02-13T21:44:42.038Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751583375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134122024-02-13T21:44:42.038Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751609776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134132024-02-13T21:44:42.038Z{"msg":"742233e3-c3a0-4f36-ab38-8cf75ce20c15 is now active with session: 4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751633257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134142024-02-13T21:44:42.039Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.751658628Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634,"session_id":"4ebe0d58-edc3-4882-9c29-fd3a6ccd97b2"}
134152024-02-13T21:44:42.039Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:40.7516802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
134162024-02-13T21:44:42.039ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
134172024-02-13T21:44:42.039ZLoading write count information from file "/tmp/test_repair_verify.data"
134182024-02-13T21:44:42.039ZRun Repair workload
134192024-02-13T21:44:42.039Z01/30 Write block 494 len 4096 data: 12
134202024-02-13T21:44:42.040Z02/30 Read block 491 len 4096
134212024-02-13T21:44:42.040Z03/30 Write block 242 len 8192 data: 12 11
134222024-02-13T21:44:42.040Z04/30 Read block 304 len 24576
134232024-02-13T21:44:42.040Z05/30 Read block 194 len 32768
134242024-02-13T21:44:42.040Z06/30 Read block 512 len 4096
134252024-02-13T21:44:42.041Z07/30 Write block 126 len 28672 data: 15 14 11 11 8 10 10
134262024-02-13T21:44:42.041Z08/30 Read block 388 len 4096
134272024-02-13T21:44:42.041Z09/30 Write block 372 len 8192 data: 16 15
134282024-02-13T21:44:42.041Z10/30 Write block 462 len 24576 data: 11 10 12 12 11 11
134292024-02-13T21:44:42.041Z11/30 Write block 486 len 12288 data: 9 8 8
134302024-02-13T21:44:42.041Z12/30 Read block 154 len 32768
134312024-02-13T21:44:42.041Z13/30 Read block 178 len 16384
134322024-02-13T21:44:42.042Z14/30 Write block 429 len 8192 data: 13 11
134332024-02-13T21:44:42.042Z15/30 Read block 321 len 40960
134342024-02-13T21:44:42.042Z16/30 Write block 172 len 40960 data: 8 7 6 7 7 10 10 9 10 9
134352024-02-13T21:44:42.042Z17/30 Read block 80 len 32768
134362024-02-13T21:44:42.042Z18/30 Write block 110 len 32768 data: 6 9 9 9 9 10 12 13
134372024-02-13T21:44:42.042Z19/30 Read block 386 len 24576
134382024-02-13T21:44:42.047Z20/30 Write block 153 len 24576 data: 9 9 10 8 8 10
134392024-02-13T21:44:42.047Z21/30 Write block 562 len 32768 data: 8 8 10 11 10 11 11 10
134402024-02-13T21:44:42.047Z22/30 Read block 549 len 40960
134412024-02-13T21:44:42.047Z23/30 Write block 40 len 28672 data: 9 9 7 6 6 6 7
134422024-02-13T21:44:42.047Z24/30 Write block 540 len 20480 data: 15 15 12 12 10
134432024-02-13T21:44:42.047Z25/30 Flush
134442024-02-13T21:44:42.048Z26/30 Write block 82 len 4096 data: 10
134452024-02-13T21:44:42.048Z27/30 Read block 585 len 28672
134462024-02-13T21:44:42.053Z28/30 Read block 224 len 8192
134472024-02-13T21:44:42.053Z29/30 Write block 382 len 24576 data: 14 12 11 11 12 11
134482024-02-13T21:44:42.053Z30/30 Read block 127 len 20480
134492024-02-13T21:44:42.053Z----------------------------------------------------------------
134502024-02-13T21:44:42.053Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134512024-02-13T21:44:42.053ZStates: Active Active Active
134522024-02-13T21:44:42.053ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134532024-02-13T21:44:42.054Z 1 Acked 1000 Write 1 Done Sent Done false
134542024-02-13T21:44:42.054Z 2 Acked 1001 Read 1 Done Sent Done false
134552024-02-13T21:44:42.054Z 3 Acked 1002 Write 2 Done Sent Done false
134562024-02-13T21:44:42.054Z 4 Acked 1003 Read 6 Done Sent Done false
134572024-02-13T21:44:42.054Z 5 Acked 1004 Read 8 Done Sent Done false
134582024-02-13T21:44:42.054Z 6 Acked 1005 Read 1 Done Sent Done false
134592024-02-13T21:44:42.055Z 7 Acked 1006 Write 7 Done Sent Done false
134602024-02-13T21:44:42.055Z 8 Acked 1007 Read 1 Done Sent Done false
134612024-02-13T21:44:42.055Z 9 Acked 1008 Write 2 Done Sent Done false
134622024-02-13T21:44:42.055Z 10 Acked 1009 Write 6 Done Sent Done false
134632024-02-13T21:44:42.055Z 11 Acked 1010 Write 3 Done Sent Done false
134642024-02-13T21:44:42.055Z 12 Acked 1011 Read 8 Done Sent Done false
134652024-02-13T21:44:42.056Z 13 Acked 1012 Read 4 Done Sent Done false
134662024-02-13T21:44:42.056Z 14 Acked 1013 Write 2 Done Sent Done false
134672024-02-13T21:44:42.056Z 15 Acked 1014 Read 10 Done Sent Done false
134682024-02-13T21:44:42.056Z 16 Acked 1015 Write 10 Done Sent Done false
134692024-02-13T21:44:42.056Z 17 Acked 1016 Read 8 Done Sent Done false
134702024-02-13T21:44:42.056Z 18 Acked 1017 Write 8 Done Sent Done false
134712024-02-13T21:44:42.056Z 19 Acked 1018 Read 6 Done Sent Done false
134722024-02-13T21:44:42.057Z 20 Acked 1019 Write 6 Done Sent Done false
134732024-02-13T21:44:42.057Z 21 Acked 1020 Write 8 Done Sent Done false
134742024-02-13T21:44:42.057Z 22 Acked 1021 Read 10 Done Sent Done false
134752024-02-13T21:44:42.057Z 23 Acked 1022 Write 7 Done Sent Done false
134762024-02-13T21:44:42.057Z 24 Acked 1023 Write 5 Done Sent Done false
134772024-02-13T21:44:42.057Z 25 Acked 1024 Flush 0 Done Sent Done false
134782024-02-13T21:44:42.057Z 26 Acked 1025 Write 1 Done Sent Done false
134792024-02-13T21:44:42.058Z 27 Acked 1026 Read 7 Done Sent Done false
134802024-02-13T21:44:42.058Z 28 Acked 1027 Read 2 Done Sent Done false
134812024-02-13T21:44:42.058Z 29 Acked 1028 Write 6 Done Sent Done false
134822024-02-13T21:44:42.058Z 30 Acked 1029 Read 5 Done Sent Sent false
134832024-02-13T21:44:42.058Z STATES DS:0 DS:1 DS:2 TOTAL
134842024-02-13T21:44:42.058Z New 0 0 0 0
134852024-02-13T21:44:42.059Z Sent 0 30 1 31
134862024-02-13T21:44:42.059Z Done 30 0 29 59
134872024-02-13T21:44:42.059Z Skipped 0 0 0 0
134882024-02-13T21:44:42.059Z Error 0 0 0 0
134892024-02-13T21:44:42.059ZLast Flush: 1024 0 1024
134902024-02-13T21:44:42.059ZDownstairs last five completed:
134912024-02-13T21:44:42.060ZUpstairs last five completed: 30 29 28 27 26
134922024-02-13T21:44:42.060Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:40.782220322Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1634}
134932024-02-13T21:44:42.060ZWrote out file "/tmp/test_repair_verify.data"
134942024-02-13T21:44:42.064Z
134952024-02-13T21:44:42.064ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134962024-02-13T21:44:42.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134972024-02-13T21:44:42.072Z 0 000-019 104 104 104 193 193 193 F F F
134982024-02-13T21:44:42.072Z 1 020-039 114 114 114 210 210 210 F F F
134992024-02-13T21:44:42.072Z 2 040-059 124 120 124 230 223 230 F F F <---
135002024-02-13T21:44:42.072Z 3 060-079 118 118 118 220 220 220 F F F
135012024-02-13T21:44:42.072Z 4 080-099 119 119 119 222 222 222 T F T <---
135022024-02-13T21:44:42.072Z 5 100-119 124 122 124 230 228 230 F F F <---
135032024-02-13T21:44:42.073Z 6 120-139 124 123 124 230 229 230 F F F <---
135042024-02-13T21:44:42.073Z 7 140-159 124 123 124 230 229 230 F F F <---
135052024-02-13T21:44:42.073Z 8 160-179 124 122 124 230 228 230 F F F <---
135062024-02-13T21:44:42.073Z 9 180-199 124 122 124 230 228 230 F F F <---
135072024-02-13T21:44:42.073Z 10 200-219 122 122 122 228 228 228 F F F
135082024-02-13T21:44:42.073Z 11 220-239 122 122 122 228 228 228 F F F
135092024-02-13T21:44:42.077Z 12 240-259 124 120 124 230 225 230 F F F <---
135102024-02-13T21:44:42.078Z 13 260-279 122 122 122 228 228 228 F F F
135112024-02-13T21:44:42.078Z 14 280-299 120 120 120 226 226 226 F F F
135122024-02-13T21:44:42.078Z 15 300-319 123 123 123 229 229 229 F F F
135132024-02-13T21:44:42.078Z 16 320-339 123 123 123 229 229 229 F F F
135142024-02-13T21:44:42.078Z 17 340-359 117 117 117 215 215 215 F F F
135152024-02-13T21:44:42.078Z 18 360-379 124 122 124 230 228 230 F F F <---
135162024-02-13T21:44:42.078Z 19 380-399 122 122 122 228 228 228 T F T <---
135172024-02-13T21:44:42.078Z 20 400-419 123 123 123 229 229 229 F F F
135182024-02-13T21:44:42.078Z 21 420-439 124 118 124 230 220 230 F F F <---
135192024-02-13T21:44:42.079Z 22 440-459 121 121 121 227 227 227 F F F
135202024-02-13T21:44:42.079Z 23 460-479 124 123 124 230 229 230 F F F <---
135212024-02-13T21:44:42.079Z 24 480-499 124 123 124 230 229 230 F F F <---
135222024-02-13T21:44:42.079Z 25 500-519 115 115 115 211 211 211 F F F
135232024-02-13T21:44:42.079Z 26 520-539 120 120 120 224 224 224 F F F
135242024-02-13T21:44:42.079Z 27 540-559 124 123 124 230 229 230 F F F <---
135252024-02-13T21:44:42.079Z 28 560-579 124 115 124 230 211 230 F F F <---
135262024-02-13T21:44:42.079Z 29 580-599 121 121 121 227 227 227 F F F
135272024-02-13T21:44:42.080ZMax gen: 124, Max flush: 230
135282024-02-13T21:44:42.080ZError: Difference in extent metadata found!
135292024-02-13T21:44:42.080ZA Difference in extent metadata is expected here
135302024-02-13T21:44:42.080ZOn loop 62
135312024-02-13T21:44:42.080Z
135322024-02-13T21:44:42.080ZVerifying data now
135332024-02-13T21:44:46.789ZNo change after verify
135342024-02-13T21:44:46.789ZLoop: 62 Downstairs dump after verify (and repair):
135352024-02-13T21:44:46.803ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135362024-02-13T21:44:46.803Z 0 000-019 104 104 104 193 193 193 F F F
135372024-02-13T21:44:46.804Z 1 020-039 114 114 114 210 210 210 F F F
135382024-02-13T21:44:46.804Z 2 040-059 124 124 124 230 230 230 F F F
135392024-02-13T21:44:46.804Z 3 060-079 118 118 118 220 220 220 F F F
135402024-02-13T21:44:46.804Z 4 080-099 125 125 125 231 231 231 F F F
135412024-02-13T21:44:46.804Z 5 100-119 124 124 124 230 230 230 F F F
135422024-02-13T21:44:46.808Z 6 120-139 124 124 124 230 230 230 F F F
135432024-02-13T21:44:46.809Z 7 140-159 124 124 124 230 230 230 F F F
135442024-02-13T21:44:46.809Z 8 160-179 124 124 124 230 230 230 F F F
135452024-02-13T21:44:46.809Z 9 180-199 124 124 124 230 230 230 F F F
135462024-02-13T21:44:46.809Z 10 200-219 122 122 122 228 228 228 F F F
135472024-02-13T21:44:46.809Z 11 220-239 122 122 122 228 228 228 F F F
135482024-02-13T21:44:46.809Z 12 240-259 124 124 124 230 230 230 F F F
135492024-02-13T21:44:46.810Z 13 260-279 122 122 122 228 228 228 F F F
135502024-02-13T21:44:46.810Z 14 280-299 120 120 120 226 226 226 F F F
135512024-02-13T21:44:46.810Z 15 300-319 123 123 123 229 229 229 F F F
135522024-02-13T21:44:46.810Z 16 320-339 123 123 123 229 229 229 F F F
135532024-02-13T21:44:46.810Z 17 340-359 117 117 117 215 215 215 F F F
135542024-02-13T21:44:46.811Z 18 360-379 124 124 124 230 230 230 F F F
135552024-02-13T21:44:46.811Z 19 380-399 125 125 125 231 231 231 F F F
135562024-02-13T21:44:46.811Z 20 400-419 123 123 123 229 229 229 F F F
135572024-02-13T21:44:46.811Z 21 420-439 124 124 124 230 230 230 F F F
135582024-02-13T21:44:46.811Z 22 440-459 121 121 121 227 227 227 F F F
135592024-02-13T21:44:46.811Z 23 460-479 124 124 124 230 230 230 F F F
135602024-02-13T21:44:46.812Z 24 480-499 124 124 124 230 230 230 F F F
135612024-02-13T21:44:46.812Z 25 500-519 115 115 115 211 211 211 F F F
135622024-02-13T21:44:46.812Z 26 520-539 120 120 120 224 224 224 F F F
135632024-02-13T21:44:46.812Z 27 540-559 124 124 124 230 230 230 F F F
135642024-02-13T21:44:46.812Z 28 560-579 124 124 124 230 230 230 F F F
135652024-02-13T21:44:46.812Z 29 580-599 121 121 121 227 227 227 F F F
135662024-02-13T21:44:46.812ZMax gen: 125, Max flush: 231
135672024-02-13T21:44:46.812Z
135682024-02-13T21:44:46.812ZBegin loop 63
135692024-02-13T21:44:46.813ZDownstairs to restart: 2
135702024-02-13T21:44:46.824Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.554589263Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
135712024-02-13T21:44:46.824Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55484544Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
135722024-02-13T21:44:46.825Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55487129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
135732024-02-13T21:44:46.832Z{"msg":"Crucible 386fa91e-7532-46ce-b756-f5be1820d355 has session id: 4e890b24-2ce2-4e69-a99b-96c684bbed86","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55490062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135742024-02-13T21:44:46.832Z{"msg":"Upstairs opts: Upstairs UUID: 386fa91e-7532-46ce-b756-f5be1820d355, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.554951281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135752024-02-13T21:44:46.832Z{"msg":"Crucible runtime is spawned
135762024-02-13T21:44:46.832ZCrucible stats registered with UUID: 386fa91e-7532-46ce-b756-f5be1820d355","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555069518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135772024-02-13T21:44:46.833Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555144881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
135782024-02-13T21:44:46.833Z{"msg":"386fa91e-7532-46ce-b756-f5be1820d355 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555220563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135792024-02-13T21:44:46.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555247494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135802024-02-13T21:44:46.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555263488Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135812024-02-13T21:44:46.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555292847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135822024-02-13T21:44:46.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555315039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135832024-02-13T21:44:46.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555338941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135842024-02-13T21:44:46.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555362892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135852024-02-13T21:44:46.834Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555380356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135862024-02-13T21:44:46.834Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555412374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135872024-02-13T21:44:46.834Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555429538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135882024-02-13T21:44:46.834Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555592409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135892024-02-13T21:44:46.834Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555646219Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135902024-02-13T21:44:46.835Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.555674009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"io task","client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135912024-02-13T21:44:46.835Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.556283428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135922024-02-13T21:44:46.835Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.556320165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135932024-02-13T21:44:46.835Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.556335929Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135942024-02-13T21:44:46.835Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.556365518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135952024-02-13T21:44:46.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.556928764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135962024-02-13T21:44:46.836Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557048511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135972024-02-13T21:44:46.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557209932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135982024-02-13T21:44:46.836Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557246619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
135992024-02-13T21:44:46.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557439039Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136002024-02-13T21:44:46.836Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557481074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136012024-02-13T21:44:46.837Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557497208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136022024-02-13T21:44:46.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557587865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136032024-02-13T21:44:46.837Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.557614685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136042024-02-13T21:44:46.837Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55762932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136052024-02-13T21:44:46.837Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55785013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136062024-02-13T21:44:46.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55829112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136072024-02-13T21:44:46.837Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55831883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136082024-02-13T21:44:46.838Z{"msg":"[0]R flush_numbers[0..12]: [193, 210, 230, 220, 231, 230, 230, 230, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558339712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136092024-02-13T21:44:46.838Z{"msg":"[0]R generation[0..12]: [104, 114, 124, 118, 125, 124, 124, 124, 124, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558360675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136102024-02-13T21:44:46.838Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558375399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136112024-02-13T21:44:46.838Z{"msg":"[1]R flush_numbers[0..12]: [193, 210, 230, 220, 231, 230, 230, 230, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558396311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136122024-02-13T21:44:46.838Z{"msg":"[1]R generation[0..12]: [104, 114, 124, 118, 125, 124, 124, 124, 124, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558411206Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136132024-02-13T21:44:46.839Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558432029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136142024-02-13T21:44:46.839Z{"msg":"[2]R flush_numbers[0..12]: [193, 210, 230, 220, 231, 230, 230, 230, 230, 230, 228, 228]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558446723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136152024-02-13T21:44:46.839Z{"msg":"[2]R generation[0..12]: [104, 114, 124, 118, 125, 124, 124, 124, 124, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558466966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136162024-02-13T21:44:46.839Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558481311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136172024-02-13T21:44:46.839Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558501633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
136182024-02-13T21:44:46.839Z,"pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136192024-02-13T21:44:46.840Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558519237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136202024-02-13T21:44:46.840Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558532102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136212024-02-13T21:44:46.840Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558552215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"":"downstairs","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136222024-02-13T21:44:46.840Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.55856572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136232024-02-13T21:44:46.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558578096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"0","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136242024-02-13T21:44:46.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558598648Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"1","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136252024-02-13T21:44:46.840Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558612083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"client":"2","":"downstairs","session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136262024-02-13T21:44:46.841Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558625588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136272024-02-13T21:44:46.841Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558644202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136282024-02-13T21:44:46.841Z{"msg":"386fa91e-7532-46ce-b756-f5be1820d355 is now active with session: 4e890b24-2ce2-4e69-a99b-96c684bbed86","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558657367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136292024-02-13T21:44:46.841Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558669672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642,"session_id":"4e890b24-2ce2-4e69-a99b-96c684bbed86"}
136302024-02-13T21:44:46.841Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:45.558688466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
136312024-02-13T21:44:46.841ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136322024-02-13T21:44:46.846ZLoading write count information from file "/tmp/test_repair_verify.data"
136332024-02-13T21:44:46.846ZRun Repair workload
136342024-02-13T21:44:46.846Z01/30 Write block 469 len 28672 data: 10 11 13 14 16 16 14
136352024-02-13T21:44:46.847Z02/30 Read block 88 len 36864
136362024-02-13T21:44:46.847Z03/30 Write block 489 len 16384 data: 8 9 8 8
136372024-02-13T21:44:46.847Z04/30 Write block 217 len 32768 data: 14 15 16 15 14 14 15 12
136382024-02-13T21:44:46.847Z05/30 Write block 53 len 4096 data: 8
136392024-02-13T21:44:46.847Z06/30 Read block 51 len 4096
136402024-02-13T21:44:46.847Z07/30 Write block 82 len 4096 data: 11
136412024-02-13T21:44:46.848Z08/30 Read block 281 len 8192
136422024-02-13T21:44:46.848Z09/30 Read block 162 len 24576
136432024-02-13T21:44:46.848Z10/30 Flush
136442024-02-13T21:44:46.848Z11/30 Write block 39 len 20480 data: 9 10 10 8 7
136452024-02-13T21:44:46.848Z12/30 Write block 180 len 8192 data: 11 10
136462024-02-13T21:44:46.848Z13/30 Write block 429 len 20480 data: 14 12 12 14 11
136472024-02-13T21:44:46.849Z14/30 Read block 87 len 28672
136482024-02-13T21:44:46.849Z15/30 Flush
136492024-02-13T21:44:46.849Z16/30 Write block 582 len 36864 data: 6 7 8 9 9 9 8 9 10
136502024-02-13T21:44:46.849Z17/30 Read block 596 len 16384
136512024-02-13T21:44:46.849Z18/30 Write block 344 len 20480 data: 12 11 10 11 10
136522024-02-13T21:44:46.854Z19/30 Read block 202 len 32768
136532024-02-13T21:44:46.854Z20/30 Flush
136542024-02-13T21:44:46.854Z21/30 Write block 456 len 24576 data: 13 13 13 13 14 11
136552024-02-13T21:44:46.854Z22/30 Write block 445 len 32768 data: 11 11 10 8 12 12 11 10
136562024-02-13T21:44:46.854Z23/30 Flush
136572024-02-13T21:44:46.858Z24/30 Write block 482 len 28672 data: 11 10 9 9 10 9 9
136582024-02-13T21:44:46.859Z25/30 Read block 572 len 20480
136592024-02-13T21:44:46.859Z26/30 Write block 389 len 16384 data: 10 8 7 7
136602024-02-13T21:44:46.859Z27/30 Write block 531 len 36864 data: 8 9 9 9 9 7 8 9 12
136612024-02-13T21:44:46.859Z28/30 Write block 65 len 36864 data: 5 5 6 9 11 11 10 12 10
136622024-02-13T21:44:46.863Z29/30 Write block 65 len 20480 data: 6 6 7 10 12
136632024-02-13T21:44:46.864Z30/30 Write block 30 len 8192 data: 8 11
136642024-02-13T21:44:46.864Z----------------------------------------------------------------
136652024-02-13T21:44:46.864Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136662024-02-13T21:44:46.864ZStates: Active Active Active
136672024-02-13T21:44:46.865ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136682024-02-13T21:44:46.865Z 1 Acked 1000 Write 7 Done Done Done false
136692024-02-13T21:44:46.865Z 2 Acked 1001 Read 9 Done Done Done false
136702024-02-13T21:44:46.865Z 3 Acked 1002 Write 4 Done Done Done false
136712024-02-13T21:44:46.865Z 4 Acked 1003 Write 8 Done Done Sent false
136722024-02-13T21:44:46.865Z 5 Acked 1004 Write 1 Done Done Sent false
136732024-02-13T21:44:46.866Z 6 Acked 1005 Read 1 Done Done Sent false
136742024-02-13T21:44:46.866Z 7 Acked 1006 Write 1 Done Done Sent false
136752024-02-13T21:44:46.866Z 8 Acked 1007 Read 2 Done Done Sent false
136762024-02-13T21:44:46.866Z 9 Acked 1008 Read 6 Done Done Sent false
136772024-02-13T21:44:46.866Z 10 Acked 1009 Flush 0 Done Done Sent false
136782024-02-13T21:44:46.866Z 11 Acked 1010 Write 5 Done Done Sent false
136792024-02-13T21:44:46.867Z 12 Acked 1011 Write 2 Done Done Sent false
136802024-02-13T21:44:46.867Z 13 Acked 1012 Write 5 Done Done Sent false
136812024-02-13T21:44:46.867Z 14 Acked 1013 Read 7 Done Done Sent false
136822024-02-13T21:44:46.867Z 15 Acked 1014 Flush 0 Done Done Sent false
136832024-02-13T21:44:46.867Z 16 Acked 1015 Write 9 Done Done Sent false
136842024-02-13T21:44:46.867Z 17 Acked 1016 Read 4 Done Done Sent false
136852024-02-13T21:44:46.867Z 18 Acked 1017 Write 5 Done Done Sent false
136862024-02-13T21:44:46.868Z 19 Acked 1018 Read 8 Done Done Sent false
136872024-02-13T21:44:46.868Z 20 Acked 1019 Flush 0 Done Done Sent false
136882024-02-13T21:44:46.868Z 21 Acked 1020 Write 6 Done Done Sent false
136892024-02-13T21:44:46.868Z 22 Acked 1021 Write 8 Done Done Sent false
136902024-02-13T21:44:46.869Z 23 Acked 1022 Flush 0 Done Done Sent false
136912024-02-13T21:44:46.869Z 24 Acked 1023 Write 7 Done Done Sent false
136922024-02-13T21:44:46.869Z 25 Acked 1024 Read 5 Done Done Sent false
136932024-02-13T21:44:46.869Z 26 Acked 1025 Write 4 Done Done Sent false
136942024-02-13T21:44:46.869Z 27 Acked 1026 Write 9 Done Done Sent false
136952024-02-13T21:44:46.869Z 28 Acked 1027 Write 9 Done Done Sent false
136962024-02-13T21:44:46.870Z 29 Acked 1028 Write 5 Sent Sent Sent false
136972024-02-13T21:44:46.870Z 30 Acked 1029 Write 2 Sent Sent Sent false
136982024-02-13T21:44:46.870Z STATES DS:0 DS:1 DS:2 TOTAL
136992024-02-13T21:44:46.870Z New 0 0 0 0
137002024-02-13T21:44:46.870Z Sent 2 2 27 31
137012024-02-13T21:44:46.870Z Done 28 28 3 59
137022024-02-13T21:44:46.870Z Skipped 0 0 0 0
137032024-02-13T21:44:46.870Z Error 0 0 0 0
137042024-02-13T21:44:46.871ZLast Flush: 1022 1022 0
137052024-02-13T21:44:46.871ZDownstairs last five completed:
137062024-02-13T21:44:46.871ZUpstairs last five completed: 30 29 28 27 26
137072024-02-13T21:44:46.871Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:45.590770787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1642}
137082024-02-13T21:44:46.871ZWrote out file "/tmp/test_repair_verify.data"
137092024-02-13T21:44:46.876Z
137102024-02-13T21:44:46.876ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
137112024-02-13T21:44:46.884ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137122024-02-13T21:44:46.884Z 0 000-019 104 104 104 193 193 193 F F F
137132024-02-13T21:44:46.884Z 1 020-039 126 126 114 233 233 210 T T F <---
137142024-02-13T21:44:46.885Z 2 040-059 126 126 124 233 233 230 F F F <---
137152024-02-13T21:44:46.885Z 3 060-079 118 118 118 220 220 220 T T F <---
137162024-02-13T21:44:46.885Z 4 080-099 126 126 125 232 232 231 F F F <---
137172024-02-13T21:44:46.889Z 5 100-119 124 124 124 230 230 230 F F F
137182024-02-13T21:44:46.889Z 6 120-139 124 124 124 230 230 230 F F F
137192024-02-13T21:44:46.889Z 7 140-159 124 124 124 230 230 230 F F F
137202024-02-13T21:44:46.889Z 8 160-179 124 124 124 230 230 230 F F F
137212024-02-13T21:44:46.889Z 9 180-199 126 126 124 233 233 230 F F F <---
137222024-02-13T21:44:46.890Z 10 200-219 126 126 122 232 232 228 F F F <---
137232024-02-13T21:44:46.890Z 11 220-239 126 126 122 232 232 228 F F F <---
137242024-02-13T21:44:46.890Z 12 240-259 124 124 124 230 230 230 F F F
137252024-02-13T21:44:46.890Z 13 260-279 122 122 122 228 228 228 F F F
137262024-02-13T21:44:46.890Z 14 280-299 120 120 120 226 226 226 F F F
137272024-02-13T21:44:46.891Z 15 300-319 123 123 123 229 229 229 F F F
137282024-02-13T21:44:46.891Z 16 320-339 123 123 123 229 229 229 F F F
137292024-02-13T21:44:46.891Z 17 340-359 126 126 117 234 234 215 F F F <---
137302024-02-13T21:44:46.891Z 18 360-379 124 124 124 230 230 230 F F F
137312024-02-13T21:44:46.892Z 19 380-399 125 125 125 231 231 231 T T F <---
137322024-02-13T21:44:46.892Z 20 400-419 123 123 123 229 229 229 F F F
137332024-02-13T21:44:46.892Z 21 420-439 126 126 124 233 233 230 F F F <---
137342024-02-13T21:44:46.892Z 22 440-459 126 126 121 235 235 227 F F F <---
137352024-02-13T21:44:46.892Z 23 460-479 126 126 124 235 235 230 F F T <---
137362024-02-13T21:44:46.892Z 24 480-499 126 126 124 232 232 230 T T T <---
137372024-02-13T21:44:46.892Z 25 500-519 115 115 115 211 211 211 F F F
137382024-02-13T21:44:46.893Z 26 520-539 120 120 120 224 224 224 T T F <---
137392024-02-13T21:44:46.893Z 27 540-559 124 124 124 230 230 230 F F F
137402024-02-13T21:44:46.893Z 28 560-579 124 124 124 230 230 230 F F F
137412024-02-13T21:44:46.893Z 29 580-599 126 126 121 234 234 227 F F F <---
137422024-02-13T21:44:46.893ZMax gen: 126, Max flush: 235
137432024-02-13T21:44:46.893ZError: Difference in extent metadata found!
137442024-02-13T21:44:46.893ZA Difference in extent metadata is expected here
137452024-02-13T21:44:46.894ZOn loop 63
137462024-02-13T21:44:46.894Z
137472024-02-13T21:44:46.898ZVerifying data now
137482024-02-13T21:44:51.590ZNo change after verify
137492024-02-13T21:44:51.590ZLoop: 63 Downstairs dump after verify (and repair):
137502024-02-13T21:44:51.604ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137512024-02-13T21:44:51.605Z 0 000-019 104 104 104 193 193 193 F F F
137522024-02-13T21:44:51.605Z 1 020-039 127 127 127 236 236 236 F F F
137532024-02-13T21:44:51.605Z 2 040-059 126 126 126 233 233 233 F F F
137542024-02-13T21:44:51.605Z 3 060-079 127 127 127 236 236 236 F F F
137552024-02-13T21:44:51.605Z 4 080-099 126 126 126 232 232 232 F F F
137562024-02-13T21:44:51.605Z 5 100-119 124 124 124 230 230 230 F F F
137572024-02-13T21:44:51.605Z 6 120-139 124 124 124 230 230 230 F F F
137582024-02-13T21:44:51.606Z 7 140-159 124 124 124 230 230 230 F F F
137592024-02-13T21:44:51.606Z 8 160-179 124 124 124 230 230 230 F F F
137602024-02-13T21:44:51.606Z 9 180-199 126 126 126 233 233 233 F F F
137612024-02-13T21:44:51.606Z 10 200-219 126 126 126 232 232 232 F F F
137622024-02-13T21:44:51.606Z 11 220-239 126 126 126 232 232 232 F F F
137632024-02-13T21:44:51.606Z 12 240-259 124 124 124 230 230 230 F F F
137642024-02-13T21:44:51.607Z 13 260-279 122 122 122 228 228 228 F F F
137652024-02-13T21:44:51.607Z 14 280-299 120 120 120 226 226 226 F F F
137662024-02-13T21:44:51.607Z 15 300-319 123 123 123 229 229 229 F F F
137672024-02-13T21:44:51.628Z 16 320-339 123 123 123 229 229 229 F F F
137682024-02-13T21:44:51.628Z 17 340-359 126 126 126 234 234 234 F F F
137692024-02-13T21:44:51.628Z 18 360-379 124 124 124 230 230 230 F F F
137702024-02-13T21:44:51.628Z 19 380-399 127 127 127 236 236 236 F F F
137712024-02-13T21:44:51.628Z 20 400-419 123 123 123 229 229 229 F F F
137722024-02-13T21:44:51.628Z 21 420-439 126 126 126 233 233 233 F F F
137732024-02-13T21:44:51.629Z 22 440-459 126 126 126 235 235 235 F F F
137742024-02-13T21:44:51.629Z 23 460-479 126 126 126 235 235 235 F F F
137752024-02-13T21:44:51.629Z 24 480-499 127 127 127 236 236 236 F F F
137762024-02-13T21:44:51.629Z 25 500-519 115 115 115 211 211 211 F F F
137772024-02-13T21:44:51.629Z 26 520-539 127 127 127 236 236 236 F F F
137782024-02-13T21:44:51.629Z 27 540-559 124 124 124 230 230 230 F F F
137792024-02-13T21:44:51.629Z 28 560-579 124 124 124 230 230 230 F F F
137802024-02-13T21:44:51.629Z 29 580-599 126 126 126 234 234 234 F F F
137812024-02-13T21:44:51.629ZMax gen: 127, Max flush: 236
137822024-02-13T21:44:51.629Z
137832024-02-13T21:44:51.630ZBegin loop 64
137842024-02-13T21:44:51.630ZDownstairs to restart: 1
137852024-02-13T21:44:51.639Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.3537845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
137862024-02-13T21:44:51.639Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354085311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
137872024-02-13T21:44:51.639ZCrucible runtime is spawned
137882024-02-13T21:44:51.639Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354140411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
137892024-02-13T21:44:51.639Z{"msg":"Crucible 18d8307a-f09e-49ea-a45d-02e4ac264a5c has session id: 9a2d7443-2e3f-42f1-b15f-f1bbe99404c8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354182076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137902024-02-13T21:44:51.639Z{"msg":"Upstairs opts: Upstairs UUID: 18d8307a-f09e-49ea-a45d-02e4ac264a5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354234847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137912024-02-13T21:44:51.639Z{"msg":"Crucible stats registered with UUID: 18d8307a-f09e-49ea-a45d-02e4ac264a5c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354277561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137922024-02-13T21:44:51.639Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354318107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
137932024-02-13T21:44:51.640Z{"msg":"18d8307a-f09e-49ea-a45d-02e4ac264a5c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354358312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137942024-02-13T21:44:51.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354400347Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137952024-02-13T21:44:51.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354443331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137962024-02-13T21:44:51.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354476629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137972024-02-13T21:44:51.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354525531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137982024-02-13T21:44:51.640Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354578482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
137992024-02-13T21:44:51.640Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354621286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138002024-02-13T21:44:51.640Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.35466545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138012024-02-13T21:44:51.640Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354786097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138022024-02-13T21:44:51.641Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354825992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138032024-02-13T21:44:51.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354857511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138042024-02-13T21:44:51.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.354889179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138052024-02-13T21:44:51.641Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.35491484Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"io task","client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138062024-02-13T21:44:51.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.355421736Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138072024-02-13T21:44:51.641Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.355470409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138082024-02-13T21:44:51.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.355497079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138092024-02-13T21:44:51.641Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.355525139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138102024-02-13T21:44:51.641Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356022698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138112024-02-13T21:44:51.642Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356052557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138122024-02-13T21:44:51.642Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356203343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138132024-02-13T21:44:51.642Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356246197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138142024-02-13T21:44:51.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356685728Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138152024-02-13T21:44:51.642Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356743177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138162024-02-13T21:44:51.642Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356769547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138172024-02-13T21:44:51.642Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356794208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138182024-02-13T21:44:51.642Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356825817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138192024-02-13T21:44:51.643Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356850828Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138202024-02-13T21:44:51.643Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.356878928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138212024-02-13T21:44:51.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.35722996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138222024-02-13T21:44:51.643Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357266737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138232024-02-13T21:44:51.643Z{"msg":"[0]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 230, 230, 230, 233, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357283361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138242024-02-13T21:44:51.643Z{"msg":"[0]R generation[0..12]: [104, 127, 126, 127, 126, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357305113Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138252024-02-13T21:44:51.643Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357319548Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138262024-02-13T21:44:51.643Z{"msg":"[1]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 230, 230, 230, 233, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.35734119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138272024-02-13T21:44:51.644Z{"msg":"[1]R generation[0..12]: [104, 127, 126, 127, 126, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357355775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138282024-02-13T21:44:51.644Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357373378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138292024-02-13T21:44:51.644Z{"msg":"[2]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 230, 230, 230, 233, 232, 232]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357387573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138302024-02-13T21:44:51.644Z{"msg":"[2]R generation[0..12]: [104, 127, 126, 127, 126, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357413454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138312024-02-13T21:44:51.644Z{Wait for a query_work_queue command to finish before sending IO
138322024-02-13T21:44:51.644Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357430218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138332024-02-13T21:44:51.644Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357454959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138342024-02-13T21:44:51.644Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357468904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138352024-02-13T21:44:51.645Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357488617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138362024-02-13T21:44:51.645Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357502332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"":"downstairs","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138372024-02-13T21:44:51.645Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357522384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138382024-02-13T21:44:51.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.35753466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"0","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138392024-02-13T21:44:51.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357548235Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"1","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138402024-02-13T21:44:51.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357568368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"client":"2","":"downstairs","session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138412024-02-13T21:44:51.645Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357581833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138422024-02-13T21:44:51.645Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357594458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138432024-02-13T21:44:51.646Z{"msg":"18d8307a-f09e-49ea-a45d-02e4ac264a5c is now active with session: 9a2d7443-2e3f-42f1-b15f-f1bbe99404c8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357607394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138442024-02-13T21:44:51.646Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357626387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650,"session_id":"9a2d7443-2e3f-42f1-b15f-f1bbe99404c8"}
138452024-02-13T21:44:51.646Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:50.357638902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
138462024-02-13T21:44:51.646ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138472024-02-13T21:44:51.646ZLoading write count information from file "/tmp/test_repair_verify.data"
138482024-02-13T21:44:51.646ZRun Repair workload
138492024-02-13T21:44:51.647Z01/30 Write block 537 len 16384 data: 9 10 13 16
138502024-02-13T21:44:51.647Z02/30 Read block 157 len 24576
138512024-02-13T21:44:51.647Z03/30 Write block 124 len 8192 data: 10 15
138522024-02-13T21:44:51.647Z04/30 Read block 441 len 32768
138532024-02-13T21:44:51.647Z05/30 Flush
138542024-02-13T21:44:51.653Z06/30 Write block 538 len 24576 data: 11 14 17 16 13 13
138552024-02-13T21:44:51.653Z07/30 Read block 261 len 16384
138562024-02-13T21:44:51.653Z08/30 Write block 142 len 8192 data: 17 14
138572024-02-13T21:44:51.653Z09/30 Read block 131 len 28672
138582024-02-13T21:44:51.653Z10/30 Read block 486 len 20480
138592024-02-13T21:44:51.653Z11/30 Write block 133 len 20480 data: 11 10 11 10 9
138602024-02-13T21:44:51.653Z12/30 Read block 123 len 40960
138612024-02-13T21:44:51.654Z13/30 Write block 262 len 4096 data: 6
138622024-02-13T21:44:51.654Z14/30 Write block 222 len 28672 data: 15 16 13 12 9 11 10
138632024-02-13T21:44:51.654Z15/30 Read block 183 len 40960
138642024-02-13T21:44:51.654Z16/30 Write block 215 len 4096 data: 11
138652024-02-13T21:44:51.654Z17/30 Flush
138662024-02-13T21:44:51.654Z18/30 Write block 448 len 16384 data: 9 13 13 12
138672024-02-13T21:44:51.655Z19/30 Read block 155 len 32768
138682024-02-13T21:44:51.655Z20/30 Read block 515 len 8192
138692024-02-13T21:44:51.655Z21/30 Read block 259 len 12288
138702024-02-13T21:44:51.655Z22/30 Read block 52 len 8192
138712024-02-13T21:44:51.655Z23/30 Read block 34 len 24576
138722024-02-13T21:44:51.655Z24/30 Write block 447 len 8192 data: 11 10
138732024-02-13T21:44:51.655Z25/30 Write block 465 len 16384 data: 13 12 12 9
138742024-02-13T21:44:51.660Z26/30 Read block 105 len 28672
138752024-02-13T21:44:51.660Z27/30 Write block 398 len 16384 data: 8 8 10 9
138762024-02-13T21:44:51.661Z28/30 Read block 287 len 12288
138772024-02-13T21:44:51.661Z29/30 Read block 139 len 36864
138782024-02-13T21:44:51.661Z30/30 Read block 24 len 12288
138792024-02-13T21:44:51.661Z----------------------------------------------------------------
138802024-02-13T21:44:51.661Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138812024-02-13T21:44:51.661ZStates: Active Active Active
138822024-02-13T21:44:51.662ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138832024-02-13T21:44:51.662Z 1 Acked 1000 Write 4 Done Sent Done false
138842024-02-13T21:44:51.662Z 2 Acked 1001 Read 6 Done Sent Done false
138852024-02-13T21:44:51.662Z 3 Acked 1002 Write 2 Done Sent Done false
138862024-02-13T21:44:51.662Z 4 Acked 1003 Read 8 Done Sent Done false
138872024-02-13T21:44:51.662Z 5 Acked 1004 Flush 0 Done Sent Done false
138882024-02-13T21:44:51.662Z 6 Acked 1005 Write 6 Done Sent Done false
138892024-02-13T21:44:51.662Z 7 Acked 1006 Read 4 Done Sent Done false
138902024-02-13T21:44:51.662Z 8 Acked 1007 Write 2 Done Sent Done false
138912024-02-13T21:44:51.662Z 9 Acked 1008 Read 7 Done Sent Done false
138922024-02-13T21:44:51.663Z 10 Acked 1009 Read 5 Done Sent Done false
138932024-02-13T21:44:51.663Z 11 Acked 1010 Write 5 Done Sent Done false
138942024-02-13T21:44:51.663Z 12 Acked 1011 Read 10 Done Sent Done false
138952024-02-13T21:44:51.663Z 13 Acked 1012 Write 1 Done Sent Done false
138962024-02-13T21:44:51.663Z 14 Acked 1013 Write 7 Done Sent Done false
138972024-02-13T21:44:51.663Z 15 Acked 1014 Read 10 Done Sent Done false
138982024-02-13T21:44:51.663Z 16 Acked 1015 Write 1 Done Sent Done false
138992024-02-13T21:44:51.664Z 17 Acked 1016 Flush 0 Done Sent Done false
139002024-02-13T21:44:51.664Z 18 Acked 1017 Write 4 Done Sent Done false
139012024-02-13T21:44:51.664Z 19 Acked 1018 Read 8 Done Sent Done false
139022024-02-13T21:44:51.664Z 20 Acked 1019 Read 2 Done Sent Done false
139032024-02-13T21:44:51.664Z 21 Acked 1020 Read 3 Done Sent Done false
139042024-02-13T21:44:51.664Z 22 Acked 1021 Read 2 Done Sent Done false
139052024-02-13T21:44:51.665Z 23 Acked 1022 Read 6 Done Sent Done false
139062024-02-13T21:44:51.665Z 24 Acked 1023 Write 2 Done Sent Done false
139072024-02-13T21:44:51.665Z 25 Acked 1024 Write 4 Done Sent Done false
139082024-02-13T21:44:51.665Z 26 Acked 1025 Read 7 Done Sent Done false
139092024-02-13T21:44:51.665Z 27 Acked 1026 Write 4 Done Sent Done false
139102024-02-13T21:44:51.665Z 28 Acked 1027 Read 3 Done Sent Done false
139112024-02-13T21:44:51.665Z 29 Acked 1028 Read 9 Done Sent Done false
139122024-02-13T21:44:51.666Z 30 Acked 1029 Read 3 Done Sent Done false
139132024-02-13T21:44:51.666Z STATES DS:0 DS:1 DS:2 TOTAL
139142024-02-13T21:44:51.666Z New 0 0 0 0
139152024-02-13T21:44:51.666Z Sent 0 30 0 30
139162024-02-13T21:44:51.666Z Done 30 0 30 60
139172024-02-13T21:44:51.666Z Skipped 0 0 0 0
139182024-02-13T21:44:51.666Z Error 0 0 0 0
139192024-02-13T21:44:51.666ZLast Flush: 1016 0 1016
139202024-02-13T21:44:51.666ZDownstairs last five completed:
139212024-02-13T21:44:51.667ZUpstairs last five completed: 30 29 28 27 26
139222024-02-13T21:44:51.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:50.38601318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1650}
139232024-02-13T21:44:51.667ZWrote out file "/tmp/test_repair_verify.data"
139242024-02-13T21:44:51.671Z
139252024-02-13T21:44:51.671ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
139262024-02-13T21:44:51.680ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139272024-02-13T21:44:51.680Z 0 000-019 104 104 104 193 193 193 F F F
139282024-02-13T21:44:51.680Z 1 020-039 127 127 127 236 236 236 F F F
139292024-02-13T21:44:51.680Z 2 040-059 126 126 126 233 233 233 F F F
139302024-02-13T21:44:51.680Z 3 060-079 127 127 127 236 236 236 F F F
139312024-02-13T21:44:51.680Z 4 080-099 126 126 126 232 232 232 F F F
139322024-02-13T21:44:51.680Z 5 100-119 124 124 124 230 230 230 F F F
139332024-02-13T21:44:51.681Z 6 120-139 128 124 128 238 230 238 F F F <---
139342024-02-13T21:44:51.681Z 7 140-159 128 124 128 238 230 238 F F F <---
139352024-02-13T21:44:51.681Z 8 160-179 124 124 124 230 230 230 F F F
139362024-02-13T21:44:51.681Z 9 180-199 126 126 126 233 233 233 F F F
139372024-02-13T21:44:51.681Z 10 200-219 128 126 128 238 232 238 F F F <---
139382024-02-13T21:44:51.681Z 11 220-239 128 126 128 238 232 238 F F F <---
139392024-02-13T21:44:51.681Z 12 240-259 124 124 124 230 230 230 F F F
139402024-02-13T21:44:51.686Z 13 260-279 128 122 128 238 228 238 F F F <---
139412024-02-13T21:44:51.686Z 14 280-299 120 120 120 226 226 226 F F F
139422024-02-13T21:44:51.686Z 15 300-319 123 123 123 229 229 229 F F F
139432024-02-13T21:44:51.686Z 16 320-339 123 123 123 229 229 229 F F F
139442024-02-13T21:44:51.686Z 17 340-359 126 126 126 234 234 234 F F F
139452024-02-13T21:44:51.686Z 18 360-379 124 124 124 230 230 230 F F F
139462024-02-13T21:44:51.686Z 19 380-399 127 127 127 236 236 236 T F T <---
139472024-02-13T21:44:51.686Z 20 400-419 123 123 123 229 229 229 T F T <---
139482024-02-13T21:44:51.686Z 21 420-439 126 126 126 233 233 233 F F F
139492024-02-13T21:44:51.687Z 22 440-459 126 126 126 235 235 235 T F T <---
139502024-02-13T21:44:51.687Z 23 460-479 126 126 126 235 235 235 T F T <---
139512024-02-13T21:44:51.687Z 24 480-499 127 127 127 236 236 236 F F F
139522024-02-13T21:44:51.687Z 25 500-519 115 115 115 211 211 211 F F F
139532024-02-13T21:44:51.687Z 26 520-539 128 127 128 238 236 238 F F F <---
139542024-02-13T21:44:51.687Z 27 540-559 128 124 128 238 230 238 F F F <---
139552024-02-13T21:44:51.687Z 28 560-579 124 124 124 230 230 230 F F F
139562024-02-13T21:44:51.687Z 29 580-599 126 126 126 234 234 234 F F F
139572024-02-13T21:44:51.687ZMax gen: 128, Max flush: 238
139582024-02-13T21:44:51.688ZError: Difference in extent metadata found!
139592024-02-13T21:44:51.688ZA Difference in extent metadata is expected here
139602024-02-13T21:44:51.688ZOn loop 64
139612024-02-13T21:44:51.688Z
139622024-02-13T21:44:51.688ZVerifying data now
139632024-02-13T21:44:56.240ZNo change after verify
139642024-02-13T21:44:56.240ZLoop: 64 Downstairs dump after verify (and repair):
139652024-02-13T21:44:56.254ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139662024-02-13T21:44:56.254Z 0 000-019 104 104 104 193 193 193 F F F
139672024-02-13T21:44:56.254Z 1 020-039 127 127 127 236 236 236 F F F
139682024-02-13T21:44:56.255Z 2 040-059 126 126 126 233 233 233 F F F
139692024-02-13T21:44:56.255Z 3 060-079 127 127 127 236 236 236 F F F
139702024-02-13T21:44:56.255Z 4 080-099 126 126 126 232 232 232 F F F
139712024-02-13T21:44:56.255Z 5 100-119 124 124 124 230 230 230 F F F
139722024-02-13T21:44:56.255Z 6 120-139 128 128 128 238 238 238 F F F
139732024-02-13T21:44:56.255Z 7 140-159 128 128 128 238 238 238 F F F
139742024-02-13T21:44:56.255Z 8 160-179 124 124 124 230 230 230 F F F
139752024-02-13T21:44:56.260Z 9 180-199 126 126 126 233 233 233 F F F
139762024-02-13T21:44:56.260Z 10 200-219 128 128 128 238 238 238 F F F
139772024-02-13T21:44:56.260Z 11 220-239 128 128 128 238 238 238 F F F
139782024-02-13T21:44:56.261Z 12 240-259 124 124 124 230 230 230 F F F
139792024-02-13T21:44:56.261Z 13 260-279 128 128 128 238 238 238 F F F
139802024-02-13T21:44:56.261Z 14 280-299 120 120 120 226 226 226 F F F
139812024-02-13T21:44:56.261Z 15 300-319 123 123 123 229 229 229 F F F
139822024-02-13T21:44:56.261Z 16 320-339 123 123 123 229 229 229 F F F
139832024-02-13T21:44:56.261Z 17 340-359 126 126 126 234 234 234 F F F
139842024-02-13T21:44:56.262Z 18 360-379 124 124 124 230 230 230 F F F
139852024-02-13T21:44:56.262Z 19 380-399 129 129 129 239 239 239 F F F
139862024-02-13T21:44:56.262Z 20 400-419 129 129 129 239 239 239 F F F
139872024-02-13T21:44:56.262Z 21 420-439 126 126 126 233 233 233 F F F
139882024-02-13T21:44:56.262Z 22 440-459 129 129 129 239 239 239 F F F
139892024-02-13T21:44:56.262Z 23 460-479 129 129 129 239 239 239 F F F
139902024-02-13T21:44:56.262Z 24 480-499 127 127 127 236 236 236 F F F
139912024-02-13T21:44:56.262Z 25 500-519 115 115 115 211 211 211 F F F
139922024-02-13T21:44:56.262Z 26 520-539 128 128 128 238 238 238 F F F
139932024-02-13T21:44:56.262Z 27 540-559 128 128 128 238 238 238 F F F
139942024-02-13T21:44:56.262Z 28 560-579 124 124 124 230 230 230 F F F
139952024-02-13T21:44:56.262Z 29 580-599 126 126 126 234 234 234 F F F
139962024-02-13T21:44:56.262ZMax gen: 129, Max flush: 239
139972024-02-13T21:44:56.262Z
139982024-02-13T21:44:56.262ZBegin loop 65
139992024-02-13T21:44:56.262ZDownstairs to restart: 0
140002024-02-13T21:44:56.275Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.001911445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
140012024-02-13T21:44:56.275Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002197671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
140022024-02-13T21:44:56.276Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
140032024-02-13T21:44:56.276Z,"time":"2024-02-13T21:44:55.00222979Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
140042024-02-13T21:44:56.282Z{"msg":"Crucible 6d8a199a-1091-43e7-8452-90ffc675070f has session id: df534a1b-604a-4d72-a211-73b75c283b31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002259979Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140052024-02-13T21:44:56.282Z{"msg":"Upstairs opts: Upstairs UUID: 6d8a199a-1091-43e7-8452-90ffc675070f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002332083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140062024-02-13T21:44:56.283Z{"msg":"Crucible stats registered with UUID: 6d8a199a-1091-43e7-8452-90ffc675070f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002364021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140072024-02-13T21:44:56.283Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002378206Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
140082024-02-13T21:44:56.283Z{"msg":"6d8a199a-1091-43e7-8452-90ffc675070f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002481258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140092024-02-13T21:44:56.283Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002510668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140102024-02-13T21:44:56.283Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002537408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140112024-02-13T21:44:56.283Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002582072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140122024-02-13T21:44:56.283Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002611651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140132024-02-13T21:44:56.283Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002650927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140142024-02-13T21:44:56.284Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002681286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140152024-02-13T21:44:56.284Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002775771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140162024-02-13T21:44:56.284Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002931985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140172024-02-13T21:44:56.284Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.002995092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140182024-02-13T21:44:56.284Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003041815Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140192024-02-13T21:44:56.284Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003077642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140202024-02-13T21:44:56.285Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:55.003112919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140212024-02-13T21:44:56.285Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:55.003140519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140222024-02-13T21:44:56.285Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:55.003163901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140232024-02-13T21:44:56.285Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003205056Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140242024-02-13T21:44:56.285Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003270092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140252024-02-13T21:44:56.285Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003300621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140262024-02-13T21:44:56.285Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003554889Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140272024-02-13T21:44:56.285Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003613048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140282024-02-13T21:44:56.285Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003658311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140292024-02-13T21:44:56.286Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.003683852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140302024-02-13T21:44:56.286Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004039993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140312024-02-13T21:44:56.286Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004193377Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140322024-02-13T21:44:56.286Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004477604Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140332024-02-13T21:44:56.286Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004513951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140342024-02-13T21:44:56.286Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004539492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140352024-02-13T21:44:56.287Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004679191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140362024-02-13T21:44:56.287Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.0047088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140372024-02-13T21:44:56.287Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:55.004732982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140382024-02-13T21:44:57.277Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.004405514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140392024-02-13T21:44:57.282Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.004658262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"io task","client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140402024-02-13T21:44:57.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.005440708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140412024-02-13T21:44:57.282Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.005478425Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140422024-02-13T21:44:57.282Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.005951983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140432024-02-13T21:44:57.283Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006479991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140442024-02-13T21:44:57.283Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006507032Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140452024-02-13T21:44:57.283Z{"msg":"[0]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 238, 238, 230, 233, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00653977Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140462024-02-13T21:44:57.283Z{"msg":"[0]R generation[0..12]: [104, 127, 126, 127, 126, 124, 128, 128, 124, 126, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006563921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140472024-02-13T21:44:57.283Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006579215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140482024-02-13T21:44:57.283Z{"msg":"[1]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 238, 238, 230, 233, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006601017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140492024-02-13T21:44:57.283Z{"msg":"[1]R generation[0..12]: [104, 127, 126, 127, 126, 124, 128, 128, 124, 126, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006615242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140502024-02-13T21:44:57.283Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006630007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140512024-02-13T21:44:57.283Z{"msg":"[2]R flush_numbers[0..12]: [193, 236, 233, 236, 232, 230, 238, 238, 230, 233, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006645481Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140522024-02-13T21:44:57.283Z{"msg":"[2]R generation[0..12]: [104, 127, 126, 127, 126, 124, 128, 128, 124, 126, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006660026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140532024-02-13T21:44:57.283Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006674271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140542024-02-13T21:44:57.283Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
140552024-02-13T21:44:57.283Z2024-02-13T21:44:56.006688156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140562024-02-13T21:44:57.283Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00670515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140572024-02-13T21:44:57.283Z{"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006718755Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140582024-02-13T21:44:57.283Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00673269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"":"downstairs","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140592024-02-13T21:44:57.283Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006746914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140602024-02-13T21:44:57.283Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.0067592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"0","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140612024-02-13T21:44:57.283Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00678598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"1","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140622024-02-13T21:44:57.283Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006799795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"client":"2","":"downstairs","session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140632024-02-13T21:44:57.283Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00681317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140642024-02-13T21:44:57.284Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006838351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140652024-02-13T21:44:57.284Z{"msg":"6d8a199a-1091-43e7-8452-90ffc675070f is now active with session: df534a1b-604a-4d72-a211-73b75c283b31","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006862652Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140662024-02-13T21:44:57.284Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.006883445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658,"session_id":"df534a1b-604a-4d72-a211-73b75c283b31"}
140672024-02-13T21:44:57.284Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:44:56.00689647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
140682024-02-13T21:44:57.284ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140692024-02-13T21:44:57.284ZLoading write count information from file "/tmp/test_repair_verify.data"
140702024-02-13T21:44:57.284ZRun Repair workload
140712024-02-13T21:44:57.287Z01/30 Write block 441 len 28672 data: 11 10 10 9 12 12 12
140722024-02-13T21:44:57.287Z02/30 Write block 243 len 4096 data: 12
140732024-02-13T21:44:57.287Z03/30 Read block 57 len 36864
140742024-02-13T21:44:57.288Z04/30 Flush
140752024-02-13T21:44:57.288Z05/30 Write block 560 len 28672 data: 8 8 9 9 11 12 11
140762024-02-13T21:44:57.288Z06/30 Read block 423 len 40960
140772024-02-13T21:44:57.291Z07/30 Flush
140782024-02-13T21:44:57.291Z08/30 Write block 19 len 40960 data: 10 11 11 12 10 10 10 8 6 7
140792024-02-13T21:44:57.294Z09/30 Write block 453 len 36864 data: 10 10 12 14 14 14 14 15 12
140802024-02-13T21:44:57.294Z10/30 Read block 189 len 4096
140812024-02-13T21:44:57.294Z11/30 Flush
140822024-02-13T21:44:57.297Z12/30 Write block 55 len 8192 data: 11 12
140832024-02-13T21:44:57.297Z13/30 Read block 157 len 16384
140842024-02-13T21:44:57.300Z14/30 Write block 509 len 36864 data: 12 13 14 13 12 15 14 13 16
140852024-02-13T21:44:57.300Z15/30 Flush
140862024-02-13T21:44:57.303Z16/30 Write block 531 len 8192 data: 9 10
140872024-02-13T21:44:57.303Z17/30 Read block 301 len 16384
140882024-02-13T21:44:57.303Z18/30 Write block 423 len 8192 data: 12 10
140892024-02-13T21:44:57.306Z19/30 Write block 174 len 28672 data: 7 8 8 11 11 10 12
140902024-02-13T21:44:57.306Z20/30 Flush
140912024-02-13T21:44:57.309Z21/30 Write block 349 len 8192 data: 8 8
140922024-02-13T21:44:57.309Z22/30 Write block 447 len 16384 data: 13 11 14 14
140932024-02-13T21:44:57.309Z23/30 Read block 326 len 12288
140942024-02-13T21:44:57.309Z24/30 Read block 400 len 16384
140952024-02-13T21:44:57.312Z25/30 Write block 373 len 28672 data: 16 14 15 14 12 12 13
140962024-02-13T21:44:57.312Z26/30 Read block 319 len 32768
140972024-02-13T21:44:57.312Z27/30 Read block 364 len 32768
140982024-02-13T21:44:57.314Z28/30 Write block 257 len 40960 data: 13 10 9 6 6 7 8 9 9 11
140992024-02-13T21:44:57.314Z29/30 Write block 416 len 20480 data: 17 19 17 15 12
141002024-02-13T21:44:57.314Z30/30 Read block 387 len 28672
141012024-02-13T21:44:57.318Z----------------------------------------------------------------
141022024-02-13T21:44:57.318Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
141032024-02-13T21:44:57.318ZStates: Active Active Active
141042024-02-13T21:44:57.318ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
141052024-02-13T21:44:57.318Z 1 Acked 1000 Write 7 Done Done Done false
141062024-02-13T21:44:57.318Z 2 Acked 1001 Write 1 Done Done Done false
141072024-02-13T21:44:57.318Z 3 Acked 1002 Read 9 Sent Done Done false
141082024-02-13T21:44:57.318Z 4 Acked 1003 Flush 0 Sent Done Done false
141092024-02-13T21:44:57.318Z 5 Acked 1004 Write 7 Sent Done Done false
141102024-02-13T21:44:57.318Z 6 Acked 1005 Read 10 Sent Done Done false
141112024-02-13T21:44:57.318Z 7 Acked 1006 Flush 0 Sent Done Done false
141122024-02-13T21:44:57.318Z 8 Acked 1007 Write 10 Sent Done Done false
141132024-02-13T21:44:57.318Z 9 Acked 1008 Write 9 Sent Done Done false
141142024-02-13T21:44:57.319Z 10 Acked 1009 Read 1 Sent Done Done false
141152024-02-13T21:44:57.319Z 11 Acked 1010 Flush 0 Sent Done Done false
141162024-02-13T21:44:57.319Z 12 Acked 1011 Write 2 Sent Done Done false
141172024-02-13T21:44:57.319Z 13 Acked 1012 Read 4 Sent Done Done false
141182024-02-13T21:44:57.319Z 14 Acked 1013 Write 9 Sent Done Done false
141192024-02-13T21:44:57.319Z 15 Acked 1014 Flush 0 Sent Done Done false
141202024-02-13T21:44:57.319Z 16 Acked 1015 Write 2 Sent Done Done false
141212024-02-13T21:44:57.319Z 17 Acked 1016 Read 4 Sent Done Done false
141222024-02-13T21:44:57.319Z 18 Acked 1017 Write 2 Sent Done Done false
141232024-02-13T21:44:57.319Z 19 Acked 1018 Write 7 Sent Done Done false
141242024-02-13T21:44:57.319Z 20 Acked 1019 Flush 0 Sent Done Done false
141252024-02-13T21:44:57.319Z 21 Acked 1020 Write 2 Sent Done Done false
141262024-02-13T21:44:57.319Z 22 Acked 1021 Write 4 Sent Done Done false
141272024-02-13T21:44:57.319Z 23 Acked 1022 Read 3 Sent Done Done false
141282024-02-13T21:44:57.319Z 24 Acked 1023 Read 4 Sent Done Done false
141292024-02-13T21:44:57.319Z 25 Acked 1024 Write 7 Sent Done Done false
141302024-02-13T21:44:57.319Z 26 Acked 1025 Read 8 Sent Done Done false
141312024-02-13T21:44:57.319Z 27 Acked 1026 Read 8 Sent Done Done false
141322024-02-13T21:44:57.319Z 28 Acked 1027 Write 10 Sent Done Done false
141332024-02-13T21:44:57.319Z 29 Acked 1028 Write 5 Sent Done Done false
141342024-02-13T21:44:57.319Z 30 Acked 1029 Read 7 Sent Done Sent false
141352024-02-13T21:44:57.319Z STATES DS:0 DS:1 DS:2 TOTAL
141362024-02-13T21:44:57.319Z New 0 0 0 0
141372024-02-13T21:44:57.319Z Sent 28 0 1 29
141382024-02-13T21:44:57.320Z Done 2 30 29 61
141392024-02-13T21:44:57.320Z Skipped 0 0 0 0
141402024-02-13T21:44:57.320Z Error 0 0 0 0
141412024-02-13T21:44:57.320ZLast Flush: 0 1019 1019
141422024-02-13T21:44:57.320ZDownstairs last five completed:
141432024-02-13T21:44:57.320ZUpstairs last five completed: 30 29 28 27 26
141442024-02-13T21:44:57.320Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:44:56.04246075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1658}
141452024-02-13T21:44:57.320ZWrote out file "/tmp/test_repair_verify.data"
141462024-02-13T21:44:57.326Z
141472024-02-13T21:44:57.326ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141482024-02-13T21:44:57.349ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141492024-02-13T21:44:57.349Z 0 000-019 104 130 130 193 242 242 F F F <---
141502024-02-13T21:44:57.349Z 1 020-039 127 130 130 236 242 242 F F F <---
141512024-02-13T21:44:57.349Z 2 040-059 126 130 130 233 243 243 F F F <---
141522024-02-13T21:44:57.349Z 3 060-079 127 127 127 236 236 236 F F F
141532024-02-13T21:44:57.349Z 4 080-099 126 126 126 232 232 232 F F F
141542024-02-13T21:44:57.349Z 5 100-119 124 124 124 230 230 230 F F F
141552024-02-13T21:44:57.349Z 6 120-139 128 128 128 238 238 238 F F F
141562024-02-13T21:44:57.349Z 7 140-159 128 128 128 238 238 238 F F F
141572024-02-13T21:44:57.349Z 8 160-179 124 130 130 230 244 244 F F F <---
141582024-02-13T21:44:57.349Z 9 180-199 126 130 130 233 244 244 F F F <---
141592024-02-13T21:44:57.349Z 10 200-219 128 128 128 238 238 238 F F F
141602024-02-13T21:44:57.350Z 11 220-239 128 128 128 238 238 238 F F F
141612024-02-13T21:44:57.350Z 12 240-259 124 130 130 230 240 240 T T T <---
141622024-02-13T21:44:57.350Z 13 260-279 128 128 128 238 238 238 F T T <---
141632024-02-13T21:44:57.350Z 14 280-299 120 120 120 226 226 226 F F F
141642024-02-13T21:44:57.350Z 15 300-319 123 123 123 229 229 229 F F F
141652024-02-13T21:44:57.350Z 16 320-339 123 123 123 229 229 229 F F F
141662024-02-13T21:44:57.350Z 17 340-359 126 126 126 234 234 234 F T T <---
141672024-02-13T21:44:57.350Z 18 360-379 124 124 124 230 230 230 F T T <---
141682024-02-13T21:44:57.350Z 19 380-399 129 129 129 239 239 239 F F F
141692024-02-13T21:44:57.350Z 20 400-419 129 129 129 239 239 239 F T T <---
141702024-02-13T21:44:57.350Z 21 420-439 126 130 130 233 244 244 F T T <---
141712024-02-13T21:44:57.350Z 22 440-459 129 130 130 239 242 242 T T T <---
141722024-02-13T21:44:57.350Z 23 460-479 129 130 130 239 242 242 F F F <---
141732024-02-13T21:44:57.350Z 24 480-499 127 127 127 236 236 236 F F F
141742024-02-13T21:44:57.350Z 25 500-519 115 130 130 211 243 243 F F F <---
141752024-02-13T21:44:57.350Z 26 520-539 128 130 130 238 244 244 F F F <---
141762024-02-13T21:44:57.350Z 27 540-559 128 128 128 238 238 238 F F F
141772024-02-13T21:44:57.350Z 28 560-579 124 130 130 230 241 241 F F F <---
141782024-02-13T21:44:57.350Z 29 580-599 126 126 126 234 234 234 F F F
141792024-02-13T21:44:57.350ZMax gen: 130, Max flush: 244
141802024-02-13T21:44:57.350ZError: Difference in extent metadata found!
141812024-02-13T21:44:57.351ZA Difference in extent metadata is expected here
141822024-02-13T21:44:57.351ZOn loop 65
141832024-02-13T21:44:57.351Z
141842024-02-13T21:44:57.351ZVerifying data now
141852024-02-13T21:45:03.119ZNo change after verify
141862024-02-13T21:45:03.119ZLoop: 65 Downstairs dump after verify (and repair):
141872024-02-13T21:45:03.134ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141882024-02-13T21:45:03.134Z 0 000-019 130 130 130 242 242 242 F F F
141892024-02-13T21:45:03.135Z 1 020-039 130 130 130 242 242 242 F F F
141902024-02-13T21:45:03.135Z 2 040-059 130 130 130 243 243 243 F F F
141912024-02-13T21:45:03.135Z 3 060-079 127 127 127 236 236 236 F F F
141922024-02-13T21:45:03.135Z 4 080-099 126 126 126 232 232 232 F F F
141932024-02-13T21:45:03.135Z 5 100-119 124 124 124 230 230 230 F F F
141942024-02-13T21:45:03.135Z 6 120-139 128 128 128 238 238 238 F F F
141952024-02-13T21:45:03.135Z 7 140-159 128 128 128 238 238 238 F F F
141962024-02-13T21:45:03.136Z 8 160-179 130 130 130 244 244 244 F F F
141972024-02-13T21:45:03.136Z 9 180-199 130 130 130 244 244 244 F F F
141982024-02-13T21:45:03.136Z 10 200-219 128 128 128 238 238 238 F F F
141992024-02-13T21:45:03.141Z 11 220-239 128 128 128 238 238 238 F F F
142002024-02-13T21:45:03.141Z 12 240-259 131 131 131 245 245 245 F F F
142012024-02-13T21:45:03.141Z 13 260-279 131 131 131 245 245 245 F F F
142022024-02-13T21:45:03.141Z 14 280-299 120 120 120 226 226 226 F F F
142032024-02-13T21:45:03.141Z 15 300-319 123 123 123 229 229 229 F F F
142042024-02-13T21:45:03.142Z 16 320-339 123 123 123 229 229 229 F F F
142052024-02-13T21:45:03.142Z 17 340-359 131 131 131 245 245 245 F F F
142062024-02-13T21:45:03.142Z 18 360-379 131 131 131 245 245 245 F F F
142072024-02-13T21:45:03.142Z 19 380-399 129 129 129 239 239 239 F F F
142082024-02-13T21:45:03.142Z 20 400-419 131 131 131 245 245 245 F F F
142092024-02-13T21:45:03.142Z 21 420-439 131 131 131 245 245 245 F F F
142102024-02-13T21:45:03.143Z 22 440-459 131 131 131 245 245 245 F F F
142112024-02-13T21:45:03.143Z 23 460-479 130 130 130 242 242 242 F F F
142122024-02-13T21:45:03.143Z 24 480-499 127 127 127 236 236 236 F F F
142132024-02-13T21:45:03.143Z 25 500-519 130 130 130 243 243 243 F F F
142142024-02-13T21:45:03.143Z 26 520-539 130 130 130 244 244 244 F F F
142152024-02-13T21:45:03.143Z 27 540-559 128 128 128 238 238 238 F F F
142162024-02-13T21:45:03.143Z 28 560-579 130 130 130 241 241 241 F F F
142172024-02-13T21:45:03.143Z 29 580-599 126 126 126 234 234 234 F F F
142182024-02-13T21:45:03.143ZMax gen: 131, Max flush: 245
142192024-02-13T21:45:03.144Z
142202024-02-13T21:45:03.144ZBegin loop 66
142212024-02-13T21:45:03.144ZDownstairs to restart: 0
142222024-02-13T21:45:03.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.878922134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
142232024-02-13T21:45:03.155Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879169614Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
142242024-02-13T21:45:03.162Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879202622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
142252024-02-13T21:45:03.163Z{"msg":"Crucible 1f2aee77-04f5-40f2-acec-87573c157d44 has session id: 6d526501-625d-4d9d-9488-574be24545ca","v":Crucible runtime is spawned
142262024-02-13T21:45:03.163Z0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879267589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142272024-02-13T21:45:03.163Z{"msg":"Upstairs opts: Upstairs UUID: 1f2aee77-04f5-40f2-acec-87573c157d44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879316891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142282024-02-13T21:45:03.163Z{"msg":"Crucible stats registered with UUID: 1f2aee77-04f5-40f2-acec-87573c157d44","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879349989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142292024-02-13T21:45:03.163Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879365793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
142302024-02-13T21:45:03.164Z{"msg":"1f2aee77-04f5-40f2-acec-87573c157d44 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879383447Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142312024-02-13T21:45:03.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879413985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142322024-02-13T21:45:03.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879439656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142332024-02-13T21:45:03.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.87948314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142342024-02-13T21:45:03.164Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879522016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142352024-02-13T21:45:03.164Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879600878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142362024-02-13T21:45:03.165Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879700641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142372024-02-13T21:45:03.165Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879740077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142382024-02-13T21:45:03.165Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879802244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142392024-02-13T21:45:03.165Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879864112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142402024-02-13T21:45:03.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879893242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142412024-02-13T21:45:03.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.87992426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142422024-02-13T21:45:03.166Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.879939775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"io task","client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142432024-02-13T21:45:03.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.880446421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142442024-02-13T21:45:03.166Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.880492324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142452024-02-13T21:45:03.166Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.880520024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142462024-02-13T21:45:03.166Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88056072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142472024-02-13T21:45:03.167Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881096335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142482024-02-13T21:45:03.167Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881250899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142492024-02-13T21:45:03.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881467201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142502024-02-13T21:45:03.167Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881512924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142512024-02-13T21:45:03.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881668458Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142522024-02-13T21:45:03.167Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881702405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142532024-02-13T21:45:03.168Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88171794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142542024-02-13T21:45:03.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881746289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142552024-02-13T21:45:03.168Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881783016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142562024-02-13T21:45:03.168Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.881815264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142572024-02-13T21:45:03.168Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88199388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142582024-02-13T21:45:03.168Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882386597Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142592024-02-13T21:45:03.169Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882418026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142602024-02-13T21:45:03.169Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 230, 238, 238, 244, 244, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88243277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142612024-02-13T21:45:03.169Z{"msg":"[0]R generation[0..12]: [130, 130, 130, 127, 126, 124, 128, 128, 130, 130, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882453733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142622024-02-13T21:45:03.169Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882468548Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142632024-02-13T21:45:03.169Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 230, 238, 238, 244, 244, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88248915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142642024-02-13T21:45:03.169Z{"msg":"[1]R generation[0..12]: [130, 130, 130, 127, 126, 124, 128, 128, 130, 130, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882503085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142652024-02-13T21:45:03.169Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882524637Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142662024-02-13T21:45:03.169Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 230, 238, 238, 244, 244, 238, 238]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882539222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142672024-02-13T21:45:03.170Z{"msg":"[2]R generation[0..12]: [130, 130, 130, 127, 126, 124, 128, 128, 130, 130, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882553577Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142682024-02-13T21:45:03.170Z{Wait for a query_work_queue command to finish before sending IO
142692024-02-13T21:45:03.170Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882577118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142702024-02-13T21:45:03.170Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882591473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142712024-02-13T21:45:03.170Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882616784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142722024-02-13T21:45:03.170Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882630089Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142732024-02-13T21:45:03.170Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882643244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"":"downstairs","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142742024-02-13T21:45:03.170Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882663267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142752024-02-13T21:45:03.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882675682Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"0","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142762024-02-13T21:45:03.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882695515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"1","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142772024-02-13T21:45:03.171Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.88270906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"client":"2","":"downstairs","session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142782024-02-13T21:45:03.171Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882722735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142792024-02-13T21:45:03.171Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882741758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142802024-02-13T21:45:03.171Z{"msg":"1f2aee77-04f5-40f2-acec-87573c157d44 is now active with session: 6d526501-625d-4d9d-9488-574be24545ca","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882755004Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142812024-02-13T21:45:03.172Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882773227Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666,"session_id":"6d526501-625d-4d9d-9488-574be24545ca"}
142822024-02-13T21:45:03.172Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:01.882785782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
142832024-02-13T21:45:03.172ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142842024-02-13T21:45:03.172ZLoading write count information from file "/tmp/test_repair_verify.data"
142852024-02-13T21:45:03.172ZRun Repair workload
142862024-02-13T21:45:03.178Z01/30 Write block 534 len 36864 data: 10 10 8 10 12 15 18 17 14
142872024-02-13T21:45:03.179Z02/30 Flush
142882024-02-13T21:45:03.179Z03/30 Write block 529 len 4096 data: 8
142892024-02-13T21:45:03.179Z04/30 Flush
142902024-02-13T21:45:03.179Z05/30 Write block 102 len 16384 data: 16 15 13 12
142912024-02-13T21:45:03.179Z06/30 Read block 536 len 8192
142922024-02-13T21:45:03.179Z07/30 Read block 219 len 32768
142932024-02-13T21:45:03.179Z08/30 Read block 100 len 40960
142942024-02-13T21:45:03.179Z09/30 Read block 563 len 8192
142952024-02-13T21:45:03.180Z10/30 Read block 118 len 24576
142962024-02-13T21:45:03.180Z11/30 Read block 46 len 36864
142972024-02-13T21:45:03.180Z12/30 Read block 90 len 8192
142982024-02-13T21:45:03.180Z13/30 Write block 456 len 32768 data: 15 15 15 15 16 13 12 11
142992024-02-13T21:45:03.180Z14/30 Write block 164 len 12288 data: 6 4 6
143002024-02-13T21:45:03.180Z15/30 Write block 543 len 16384 data: 14 11 12 11
143012024-02-13T21:45:03.181Z16/30 Flush
143022024-02-13T21:45:03.186Z17/30 Write block 179 len 8192 data: 11 13
143032024-02-13T21:45:03.186Z18/30 Read block 207 len 20480
143042024-02-13T21:45:03.186Z19/30 Write block 130 len 20480 data: 9 11 11 12 11
143052024-02-13T21:45:03.186Z20/30 Write block 489 len 4096 data: 9
143062024-02-13T21:45:03.186Z21/30 Read block 486 len 32768
143072024-02-13T21:45:03.186Z22/30 Read block 259 len 12288
143082024-02-13T21:45:03.186Z23/30 Flush
143092024-02-13T21:45:03.187Z24/30 Write block 521 len 28672 data: 17 15 13 9 6 6 8
143102024-02-13T21:45:03.187Z25/30 Read block 573 len 4096
143112024-02-13T21:45:03.191Z26/30 Write block 297 len 32768 data: 10 10 9 8 8 7 9 9
143122024-02-13T21:45:03.191Z27/30 Read block 554 len 12288
143132024-02-13T21:45:03.192Z28/30 Read block 150 len 24576
143142024-02-13T21:45:03.192Z29/30 Write block 222 len 12288 data: 16 17 14
143152024-02-13T21:45:03.192Z30/30 Write block 350 len 36864 data: 9 9 10 11 11 12 15 14 13
143162024-02-13T21:45:03.192Z----------------------------------------------------------------
143172024-02-13T21:45:03.192Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:28
143182024-02-13T21:45:03.192ZStates: Active Active Active
143192024-02-13T21:45:03.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143202024-02-13T21:45:03.193Z 3 Acked 1002 Write 1 Sent Done Done false
143212024-02-13T21:45:03.193Z 4 Acked 1003 Flush 0 Sent Done Done false
143222024-02-13T21:45:03.193Z 5 Acked 1004 Write 4 Sent Done Done false
143232024-02-13T21:45:03.193Z 6 Acked 1005 Read 2 Sent Done Done false
143242024-02-13T21:45:03.193Z 7 Acked 1006 Read 8 Sent Done Done false
143252024-02-13T21:45:03.193Z 8 Acked 1007 Read 10 Sent Done Done false
143262024-02-13T21:45:03.193Z 9 Acked 1008 Read 2 Sent Done Done false
143272024-02-13T21:45:03.194Z 10 Acked 1009 Read 6 Sent Done Done false
143282024-02-13T21:45:03.194Z 11 Acked 1010 Read 9 Sent Done Done false
143292024-02-13T21:45:03.194Z 12 Acked 1011 Read 2 Sent Done Done false
143302024-02-13T21:45:03.194Z 13 Acked 1012 Write 8 Sent Done Done false
143312024-02-13T21:45:03.194Z 14 Acked 1013 Write 3 Sent Done Done false
143322024-02-13T21:45:03.194Z 15 Acked 1014 Write 4 Sent Done Done false
143332024-02-13T21:45:03.194Z 16 Acked 1015 Flush 0 Sent Done Done false
143342024-02-13T21:45:03.194Z 17 Acked 1016 Write 2 Sent Done Done false
143352024-02-13T21:45:03.195Z 18 Acked 1017 Read 5 Sent Done Done false
143362024-02-13T21:45:03.195Z 19 Acked 1018 Write 5 Sent Done Done false
143372024-02-13T21:45:03.195Z 20 Acked 1019 Write 1 Sent Done Done false
143382024-02-13T21:45:03.195Z 21 Acked 1020 Read 8 Sent Done Done false
143392024-02-13T21:45:03.195Z 22 Acked 1021 Read 3 Sent Done Done false
143402024-02-13T21:45:03.196Z 23 Acked 1022 Flush 0 Sent Done Done false
143412024-02-13T21:45:03.196Z 24 Acked 1023 Write 7 Sent Done Done false
143422024-02-13T21:45:03.196Z 25 Acked 1024 Read 1 Sent Done Done false
143432024-02-13T21:45:03.196Z 26 Acked 1025 Write 8 Sent Done Done false
143442024-02-13T21:45:03.196Z 27 Acked 1026 Read 3 Sent Done Done false
143452024-02-13T21:45:03.196Z 28 Acked 1027 Read 6 Sent Done Done false
143462024-02-13T21:45:03.197Z 29 Acked 1028 Write 3 Sent Done Done false
143472024-02-13T21:45:03.197Z 30 Acked 1029 Write 9 Sent Sent Sent false
143482024-02-13T21:45:03.197Z STATES DS:0 DS:1 DS:2 TOTAL
143492024-02-13T21:45:03.197Z New 0 0 0 0
143502024-02-13T21:45:03.197Z Sent 28 1 1 30
143512024-02-13T21:45:03.197Z Done 0 27 27 54
143522024-02-13T21:45:03.197Z Skipped 0 0 0 0
143532024-02-13T21:45:03.198Z Error 0 0 0 0
143542024-02-13T21:45:03.198ZLast Flush: 1001 1022 1022
143552024-02-13T21:45:03.198ZDownstairs last five completed: 1001 1000
143562024-02-13T21:45:03.198ZUpstairs last five completed: 30 29 28 27 26
143572024-02-13T21:45:03.198Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:01.915034385Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1666}
143582024-02-13T21:45:03.203ZWrote out file "/tmp/test_repair_verify.data"
143592024-02-13T21:45:03.204Z
143602024-02-13T21:45:03.204ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143612024-02-13T21:45:03.215ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143622024-02-13T21:45:03.215Z 0 000-019 130 130 130 242 242 242 F F F
143632024-02-13T21:45:03.215Z 1 020-039 130 130 130 242 242 242 F F F
143642024-02-13T21:45:03.216Z 2 040-059 130 130 130 243 243 243 F F F
143652024-02-13T21:45:03.216Z 3 060-079 127 127 127 236 236 236 F F F
143662024-02-13T21:45:03.216Z 4 080-099 126 126 126 232 232 232 F F F
143672024-02-13T21:45:03.216Z 5 100-119 124 132 132 230 248 248 F F F <---
143682024-02-13T21:45:03.216Z 6 120-139 128 132 132 238 249 249 F F F <---
143692024-02-13T21:45:03.216Z 7 140-159 128 128 128 238 238 238 F F F
143702024-02-13T21:45:03.217Z 8 160-179 130 132 132 244 249 249 F F F <---
143712024-02-13T21:45:03.217Z 9 180-199 130 132 132 244 249 249 F F F <---
143722024-02-13T21:45:03.217Z 10 200-219 128 128 128 238 238 238 F F F
143732024-02-13T21:45:03.222Z 11 220-239 128 128 128 238 238 238 F T T <---
143742024-02-13T21:45:03.222Z 12 240-259 131 131 131 245 245 245 F F F
143752024-02-13T21:45:03.222Z 13 260-279 131 131 131 245 245 245 F F F
143762024-02-13T21:45:03.222Z 14 280-299 120 120 120 226 226 226 F T T <---
143772024-02-13T21:45:03.222Z 15 300-319 123 123 123 229 229 229 F T T <---
143782024-02-13T21:45:03.223Z 16 320-339 123 123 123 229 229 229 F F F
143792024-02-13T21:45:03.223Z 17 340-359 131 131 131 245 245 245 F T T <---
143802024-02-13T21:45:03.223Z 18 360-379 131 131 131 245 245 245 F F F
143812024-02-13T21:45:03.223Z 19 380-399 129 129 129 239 239 239 F F F
143822024-02-13T21:45:03.223Z 20 400-419 131 131 131 245 245 245 F F F
143832024-02-13T21:45:03.223Z 21 420-439 131 131 131 245 245 245 F F F
143842024-02-13T21:45:03.223Z 22 440-459 131 132 132 245 248 248 F F F <---
143852024-02-13T21:45:03.224Z 23 460-479 130 132 132 242 248 248 F F F <---
143862024-02-13T21:45:03.224Z 24 480-499 127 132 132 236 249 249 F F F <---
143872024-02-13T21:45:03.224Z 25 500-519 130 130 130 243 243 243 F F F
143882024-02-13T21:45:03.224Z 26 520-539 132 132 132 246 247 247 F T T <---
143892024-02-13T21:45:03.224Z 27 540-559 132 132 132 246 248 248 F F F <---
143902024-02-13T21:45:03.224Z 28 560-579 130 130 130 241 241 241 F F F
143912024-02-13T21:45:03.224Z 29 580-599 126 126 126 234 234 234 F F F
143922024-02-13T21:45:03.224ZMax gen: 132, Max flush: 249
143932024-02-13T21:45:03.225ZError: Difference in extent metadata found!
143942024-02-13T21:45:03.225ZA Difference in extent metadata is expected here
143952024-02-13T21:45:03.225ZOn loop 66
143962024-02-13T21:45:03.225Z
143972024-02-13T21:45:03.225ZVerifying data now
143982024-02-13T21:45:07.862ZNo change after verify
143992024-02-13T21:45:07.862ZLoop: 66 Downstairs dump after verify (and repair):
144002024-02-13T21:45:07.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144012024-02-13T21:45:07.876Z 0 000-019 130 130 130 242 242 242 F F F
144022024-02-13T21:45:07.876Z 1 020-039 130 130 130 242 242 242 F F F
144032024-02-13T21:45:07.876Z 2 040-059 130 130 130 243 243 243 F F F
144042024-02-13T21:45:07.876Z 3 060-079 127 127 127 236 236 236 F F F
144052024-02-13T21:45:07.876Z 4 080-099 126 126 126 232 232 232 F F F
144062024-02-13T21:45:07.877Z 5 100-119 132 132 132 248 248 248 F F F
144072024-02-13T21:45:07.877Z 6 120-139 132 132 132 249 249 249 F F F
144082024-02-13T21:45:07.877Z 7 140-159 128 128 128 238 238 238 F F F
144092024-02-13T21:45:07.877Z 8 160-179 132 132 132 249 249 249 F F F
144102024-02-13T21:45:07.941Z 9 180-199 132 132 132 249 249 249 F F F
144112024-02-13T21:45:07.941Z 10 200-219 128 128 128 238 238 238 F F F
144122024-02-13T21:45:07.941Z 11 220-239 133 133 133 250 250 250 F F F
144132024-02-13T21:45:07.941Z 12 240-259 131 131 131 245 245 245 F F F
144142024-02-13T21:45:07.941Z 13 260-279 131 131 131 245 245 245 F F F
144152024-02-13T21:45:07.941Z 14 280-299 133 133 133 250 250 250 F F F
144162024-02-13T21:45:07.942Z 15 300-319 133 133 133 250 250 250 F F F
144172024-02-13T21:45:07.942Z 16 320-339 123 123 123 229 229 229 F F F
144182024-02-13T21:45:07.942Z 17 340-359 133 133 133 250 250 250 F F F
144192024-02-13T21:45:07.942Z 18 360-379 131 131 131 245 245 245 F F F
144202024-02-13T21:45:07.942Z 19 380-399 129 129 129 239 239 239 F F F
144212024-02-13T21:45:07.942Z 20 400-419 131 131 131 245 245 245 F F F
144222024-02-13T21:45:07.942Z 21 420-439 131 131 131 245 245 245 F F F
144232024-02-13T21:45:07.942Z 22 440-459 132 132 132 248 248 248 F F F
144242024-02-13T21:45:07.942Z 23 460-479 132 132 132 248 248 248 F F F
144252024-02-13T21:45:07.942Z 24 480-499 132 132 132 249 249 249 F F F
144262024-02-13T21:45:07.942Z 25 500-519 130 130 130 243 243 243 F F F
144272024-02-13T21:45:07.943Z 26 520-539 133 133 133 250 250 250 F F F
144282024-02-13T21:45:07.943Z 27 540-559 132 132 132 248 248 248 F F F
144292024-02-13T21:45:07.943Z 28 560-579 130 130 130 241 241 241 F F F
144302024-02-13T21:45:07.943Z 29 580-599 126 126 126 234 234 234 F F F
144312024-02-13T21:45:07.943ZMax gen: 133, Max flush: 250
144322024-02-13T21:45:07.943Z
144332024-02-13T21:45:07.943ZBegin loop 67
144342024-02-13T21:45:07.943ZDownstairs to restart: 2
144352024-02-13T21:45:07.943Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619100779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674}
144362024-02-13T21:45:07.943Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619370301Z","hostname":"Crucible runtime is spawned
144372024-02-13T21:45:07.943Zip-10-150-1-148.us-west-2.compute.internal","pid":1674}
144382024-02-13T21:45:07.943Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619464936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674}
144392024-02-13T21:45:07.944Z{"msg":"Crucible 2f7d60cc-ca26-4902-bcb7-f3d981046abf has session id: fc1c4739-9ca9-48e7-8e39-5dc4da9ade56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619504902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144402024-02-13T21:45:07.944Z{"msg":"Upstairs opts: Upstairs UUID: 2f7d60cc-ca26-4902-bcb7-f3d981046abf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619541679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144412024-02-13T21:45:07.944Z{"msg":"Crucible stats registered with UUID: 2f7d60cc-ca26-4902-bcb7-f3d981046abf","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619585473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144422024-02-13T21:45:07.944Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.61961964Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674}
144432024-02-13T21:45:07.944Z{"msg":"2f7d60cc-ca26-4902-bcb7-f3d981046abf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619651809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144442024-02-13T21:45:07.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619680328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144452024-02-13T21:45:07.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619724152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144462024-02-13T21:45:07.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619769256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144472024-02-13T21:45:07.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619797746Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144482024-02-13T21:45:07.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619830984Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144492024-02-13T21:45:07.945Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619855155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144502024-02-13T21:45:07.945Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619917792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144512024-02-13T21:45:07.945Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619957378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144522024-02-13T21:45:07.945Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.619992855Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144532024-02-13T21:45:07.945Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620016926Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144542024-02-13T21:45:07.945Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620040638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144552024-02-13T21:45:07.945Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620065169Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"io task","client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144562024-02-13T21:45:07.945Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620720461Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144572024-02-13T21:45:07.945Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620765975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144582024-02-13T21:45:07.945Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620789596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144592024-02-13T21:45:07.945Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.620811978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144602024-02-13T21:45:07.945Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621331919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144612024-02-13T21:45:07.946Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621382021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144622024-02-13T21:45:07.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621874042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144632024-02-13T21:45:07.946Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621910989Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144642024-02-13T21:45:07.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621927693Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144652024-02-13T21:45:07.946Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.621989461Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144662024-02-13T21:45:07.946Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622016511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144672024-02-13T21:45:07.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622031076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144682024-02-13T21:45:07.946Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622044571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144692024-02-13T21:45:07.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622065093Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144702024-02-13T21:45:07.946Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622556355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144712024-02-13T21:45:07.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.622981731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144722024-02-13T21:45:07.946Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623015519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144732024-02-13T21:45:07.946Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 248, 249, 238, 249, 249, 238, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623030873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144742024-02-13T21:45:07.947Z{"msg":"[0]R generation[0..12]: [130, 130, 130, 127, 126, 132, 132, 128, 132, 132, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623045877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144752024-02-13T21:45:07.947Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.62306731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144762024-02-13T21:45:07.947Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 248, 249, 238, 249, 249, 238, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623081715Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144772024-02-13T21:45:07.947Z{"msg":"[1]R generation[0..12]: [130, 130, 130, 127, 126, 132, 132, 128, 132, 132, 128, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623102177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144782024-02-13T21:45:07.947Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623116812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144792024-02-13T21:45:07.947Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 243, 236, 232, 248, 249, 238, 249, 249, 238, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623137464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144802024-02-13T21:45:07.947Z{"msg":"[2]R generation[0..12]: [130, 130, 130, 127, 126, 132, 132, 128, 132, 132, 128, 133]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
144812024-02-13T21:45:07.947Zcrucible","level":30,"time":"2024-02-13T21:45:06.623166274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144822024-02-13T21:45:07.947Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623181318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144832024-02-13T21:45:07.947Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623200981Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144842024-02-13T21:45:07.947Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623215366Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144852024-02-13T21:45:07.948Z{"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623235209Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144862024-02-13T21:45:07.948Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623248764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"":"downstairs","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144872024-02-13T21:45:07.948Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623268537Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144882024-02-13T21:45:07.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623280982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"0","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144892024-02-13T21:45:07.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623300725Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"1","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144902024-02-13T21:45:07.948Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.62331446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"client":"2","":"downstairs","session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144912024-02-13T21:45:07.948Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623334453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144922024-02-13T21:45:07.948Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623347088Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144932024-02-13T21:45:07.948Z{"msg":"2f7d60cc-ca26-4902-bcb7-f3d981046abf is now active with session: fc1c4739-9ca9-48e7-8e39-5dc4da9ade56","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623366291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144942024-02-13T21:45:07.948Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.623378297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674,"session_id":"fc1c4739-9ca9-48e7-8e39-5dc4da9ade56"}
144952024-02-13T21:45:07.948Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:06.62339687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674}
144962024-02-13T21:45:07.948ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144972024-02-13T21:45:07.949ZLoading write count information from file "/tmp/test_repair_verify.data"
144982024-02-13T21:45:07.949ZRun Repair workload
144992024-02-13T21:45:07.949Z01/30 Write block 192 len 12288 data: 10 11 13
145002024-02-13T21:45:07.949Z02/30 Read block 240 len 32768
145012024-02-13T21:45:07.949Z03/30 Write block 353 len 12288 data: 12 12 13
145022024-02-13T21:45:07.949Z04/30 Read block 211 len 40960
145032024-02-13T21:45:07.949Z05/30 Read block 86 len 40960
145042024-02-13T21:45:07.977Z06/30 Write block 365 len 36864 data: 9 9 13 12 12 11 13 17 17
145052024-02-13T21:45:07.977Z07/30 Read block 194 len 16384
145062024-02-13T21:45:07.977Z08/30 Read block 62 len 24576
145072024-02-13T21:45:07.977Z09/30 Write block 400 len 12288 data: 11 10 10
145082024-02-13T21:45:07.977Z10/30 Write block 198 len 16384 data: 15 14 12 12
145092024-02-13T21:45:07.977Z11/30 Write block 408 len 8192 data: 9 12
145102024-02-13T21:45:07.977Z12/30 Read block 520 len 32768
145112024-02-13T21:45:07.978Z13/30 Write block 62 len 36864 data: 10 8 5 7 7 8 11 13 12
145122024-02-13T21:45:07.978Z14/30 Write block 205 len 4096 data: 7
145132024-02-13T21:45:07.978Z15/30 Read block 240 len 12288
145142024-02-13T21:45:07.978Z16/30 Flush
145152024-02-13T21:45:07.978Z17/30 Write block 85 len 28672 data: 10 9 8 8 8 7 8
145162024-02-13T21:45:07.978Z18/30 Flush
145172024-02-13T21:45:07.978Z19/30 Write block 365 len 8192 data: 10 10
145182024-02-13T21:45:07.978Z20/30 Flush
145192024-02-13T21:45:07.978Z21/30 Write block 364 len 40960 data: 10 11 11 14 13 13 12 14 18 18
145202024-02-13T21:45:07.978Z22/30 Read block 169 len 12288
145212024-02-13T21:45:07.979Z23/30 Write block 354 len 40960 data: 13 14 16 15 14 12 12 10 10 12
145222024-02-13T21:45:07.979Z24/30 Write block 533 len 28672 data: 10 11 11 9 11 13 16
145232024-02-13T21:45:07.979Z25/30 Read block 433 len 8192
145242024-02-13T21:45:07.979Z26/30 Read block 434 len 20480
145252024-02-13T21:45:07.979Z27/30 Read block 4 len 12288
145262024-02-13T21:45:07.979Z28/30 Read block 389 len 20480
145272024-02-13T21:45:07.979Z29/30 Write block 513 len 36864 data: 13 16 15 14 17 16 19 21 18
145282024-02-13T21:45:07.979Z30/30 Read block 485 len 8192
145292024-02-13T21:45:07.980Z----------------------------------------------------------------
145302024-02-13T21:45:07.980Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
145312024-02-13T21:45:07.980ZStates: Active Active Active
145322024-02-13T21:45:07.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145332024-02-13T21:45:07.980Z 1 Acked 1000 Write 3 Done Done Sent false
145342024-02-13T21:45:07.980Z 2 Acked 1001 Read 8 Done Done Sent false
145352024-02-13T21:45:07.980Z 3 Acked 1002 Write 3 Done Done Sent false
145362024-02-13T21:45:07.980Z 4 Acked 1003 Read 10 Done Done Sent false
145372024-02-13T21:45:07.980Z 5 Acked 1004 Read 10 Done Done Sent false
145382024-02-13T21:45:07.980Z 6 Acked 1005 Write 9 Done Done Sent false
145392024-02-13T21:45:07.981Z 7 Acked 1006 Read 4 Done Done Sent false
145402024-02-13T21:45:07.981Z 8 Acked 1007 Read 6 Done Done Sent false
145412024-02-13T21:45:07.981Z 9 Acked 1008 Write 3 Done Done Sent false
145422024-02-13T21:45:07.981Z 10 Acked 1009 Write 4 Done Done Sent false
145432024-02-13T21:45:07.981Z 11 Acked 1010 Write 2 Done Done Sent false
145442024-02-13T21:45:07.981Z 12 Acked 1011 Read 8 Done Done Sent false
145452024-02-13T21:45:07.981Z 13 Acked 1012 Write 9 Done Done Sent false
145462024-02-13T21:45:07.981Z 14 Acked 1013 Write 1 Done Done Sent false
145472024-02-13T21:45:07.981Z 15 Acked 1014 Read 3 Done Done Sent false
145482024-02-13T21:45:07.981Z 16 Acked 1015 Flush 0 Done Done Sent false
145492024-02-13T21:45:07.986Z 17 Acked 1016 Write 7 Done Done Sent false
145502024-02-13T21:45:07.986Z 18 Acked 1017 Flush 0 Done Done Sent false
145512024-02-13T21:45:07.986Z 19 Acked 1018 Write 2 Done Done Sent false
145522024-02-13T21:45:07.986Z 20 Acked 1019 Flush 0 Done Done Sent false
145532024-02-13T21:45:07.987Z 21 Acked 1020 Write 10 Done Done Sent false
145542024-02-13T21:45:07.987Z 22 Acked 1021 Read 3 Done Done Sent false
145552024-02-13T21:45:07.987Z 23 Acked 1022 Write 10 Done Done Sent false
145562024-02-13T21:45:07.987Z 24 Acked 1023 Write 7 Done Done Sent false
145572024-02-13T21:45:07.987Z 25 Acked 1024 Read 2 Done Done Sent false
145582024-02-13T21:45:07.987Z 26 Acked 1025 Read 5 Done Done Sent false
145592024-02-13T21:45:07.987Z 27 Acked 1026 Read 3 Done Done Sent false
145602024-02-13T21:45:07.987Z 28 Acked 1027 Read 5 Done Done Sent false
145612024-02-13T21:45:07.987Z 29 Acked 1028 Write 9 Done Done Sent false
145622024-02-13T21:45:07.988Z 30 Acked 1029 Read 2 Done Done Sent false
145632024-02-13T21:45:07.988Z STATES DS:0 DS:1 DS:2 TOTAL
145642024-02-13T21:45:07.988Z New 0 0 0 0
145652024-02-13T21:45:07.988Z Sent 0 0 30 30
145662024-02-13T21:45:07.988Z Done 30 30 0 60
145672024-02-13T21:45:07.988Z Skipped 0 0 0 0
145682024-02-13T21:45:07.988Z Error 0 0 0 0
145692024-02-13T21:45:07.988ZLast Flush: 1019 1019 0
145702024-02-13T21:45:07.988ZDownstairs last five completed:
145712024-02-13T21:45:07.988ZUpstairs last five completed: 30 29 28 27 26
145722024-02-13T21:45:07.988Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:06.656197992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1674}
145732024-02-13T21:45:07.988ZWrote out file "/tmp/test_repair_verify.data"
145742024-02-13T21:45:07.988Z
145752024-02-13T21:45:07.989ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145762024-02-13T21:45:07.989ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145772024-02-13T21:45:07.989Z 0 000-019 130 130 130 242 242 242 F F F
145782024-02-13T21:45:07.989Z 1 020-039 130 130 130 242 242 242 F F F
145792024-02-13T21:45:07.989Z 2 040-059 130 130 130 243 243 243 F F F
145802024-02-13T21:45:07.989Z 3 060-079 134 134 127 251 251 236 F F F <---
145812024-02-13T21:45:07.989Z 4 080-099 134 134 126 252 252 232 F F F <---
145822024-02-13T21:45:07.989Z 5 100-119 132 132 132 248 248 248 F F F
145832024-02-13T21:45:07.989Z 6 120-139 132 132 132 249 249 249 F F F
145842024-02-13T21:45:07.989Z 7 140-159 128 128 128 238 238 238 F F F
145852024-02-13T21:45:07.989Z 8 160-179 132 132 132 249 249 249 F F F
145862024-02-13T21:45:07.989Z 9 180-199 134 134 132 251 251 249 F F F <---
145872024-02-13T21:45:07.989Z 10 200-219 134 134 128 251 251 238 F F F <---
145882024-02-13T21:45:07.989Z 11 220-239 133 133 133 250 250 250 F F F
145892024-02-13T21:45:07.989Z 12 240-259 131 131 131 245 245 245 F F F
145902024-02-13T21:45:07.989Z 13 260-279 131 131 131 245 245 245 F F F
145912024-02-13T21:45:07.989Z 14 280-299 133 133 133 250 250 250 F F F
145922024-02-13T21:45:07.989Z 15 300-319 133 133 133 250 250 250 F F F
145932024-02-13T21:45:07.989Z 16 320-339 123 123 123 229 229 229 F F F
145942024-02-13T21:45:07.989Z 17 340-359 134 134 133 251 251 250 T T F <---
145952024-02-13T21:45:07.989Z 18 360-379 134 134 131 253 253 245 T T F <---
145962024-02-13T21:45:07.990Z 19 380-399 129 129 129 239 239 239 F F F
145972024-02-13T21:45:07.990Z 20 400-419 134 134 131 251 251 245 F F F <---
145982024-02-13T21:45:07.990Z 21 420-439 131 131 131 245 245 245 F F F
145992024-02-13T21:45:07.990Z 22 440-459 132 132 132 248 248 248 F F F
146002024-02-13T21:45:07.990Z 23 460-479 132 132 132 248 248 248 F F F
146012024-02-13T21:45:07.990Z 24 480-499 132 132 132 249 249 249 F F F
146022024-02-13T21:45:07.990Z 25 500-519 130 130 130 243 243 243 T T F <---
146032024-02-13T21:45:07.990Z 26 520-539 133 133 133 250 250 250 T T F <---
146042024-02-13T21:45:07.994Z 27 540-559 132 132 132 248 248 248 F F F
146052024-02-13T21:45:07.994Z 28 560-579 130 130 130 241 241 241 F F F
146062024-02-13T21:45:07.995Z 29 580-599 126 126 126 234 234 234 F F F
146072024-02-13T21:45:07.995ZMax gen: 134, Max flush: 253
146082024-02-13T21:45:07.995ZError: Difference in extent metadata found!
146092024-02-13T21:45:07.995ZA Difference in extent metadata is expected here
146102024-02-13T21:45:07.995ZOn loop 67
146112024-02-13T21:45:07.995Z
146122024-02-13T21:45:07.995ZVerifying data now
146132024-02-13T21:45:12.446ZNo change after verify
146142024-02-13T21:45:12.446ZLoop: 67 Downstairs dump after verify (and repair):
146152024-02-13T21:45:12.460ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146162024-02-13T21:45:12.460Z 0 000-019 130 130 130 242 242 242 F F F
146172024-02-13T21:45:12.460Z 1 020-039 130 130 130 242 242 242 F F F
146182024-02-13T21:45:12.461Z 2 040-059 130 130 130 243 243 243 F F F
146192024-02-13T21:45:12.465Z 3 060-079 134 134 134 251 251 251 F F F
146202024-02-13T21:45:12.465Z 4 080-099 134 134 134 252 252 252 F F F
146212024-02-13T21:45:12.465Z 5 100-119 132 132 132 248 248 248 F F F
146222024-02-13T21:45:12.465Z 6 120-139 132 132 132 249 249 249 F F F
146232024-02-13T21:45:12.466Z 7 140-159 128 128 128 238 238 238 F F F
146242024-02-13T21:45:12.466Z 8 160-179 132 132 132 249 249 249 F F F
146252024-02-13T21:45:12.466Z 9 180-199 134 134 134 251 251 251 F F F
146262024-02-13T21:45:12.466Z 10 200-219 134 134 134 251 251 251 F F F
146272024-02-13T21:45:12.466Z 11 220-239 133 133 133 250 250 250 F F F
146282024-02-13T21:45:12.466Z 12 240-259 131 131 131 245 245 245 F F F
146292024-02-13T21:45:12.466Z 13 260-279 131 131 131 245 245 245 F F F
146302024-02-13T21:45:12.467Z 14 280-299 133 133 133 250 250 250 F F F
146312024-02-13T21:45:12.467Z 15 300-319 133 133 133 250 250 250 F F F
146322024-02-13T21:45:12.467Z 16 320-339 123 123 123 229 229 229 F F F
146332024-02-13T21:45:12.467Z 17 340-359 135 135 135 254 254 254 F F F
146342024-02-13T21:45:12.467Z 18 360-379 135 135 135 254 254 254 F F F
146352024-02-13T21:45:12.467Z 19 380-399 129 129 129 239 239 239 F F F
146362024-02-13T21:45:12.468Z 20 400-419 134 134 134 251 251 251 F F F
146372024-02-13T21:45:12.468Z 21 420-439 131 131 131 245 245 245 F F F
146382024-02-13T21:45:12.468Z 22 440-459 132 132 132 248 248 248 F F F
146392024-02-13T21:45:12.468Z 23 460-479 132 132 132 248 248 248 F F F
146402024-02-13T21:45:12.468Z 24 480-499 132 132 132 249 249 249 F F F
146412024-02-13T21:45:12.468Z 25 500-519 135 135 135 254 254 254 F F F
146422024-02-13T21:45:12.468Z 26 520-539 135 135 135 254 254 254 F F F
146432024-02-13T21:45:12.469Z 27 540-559 132 132 132 248 248 248 F F F
146442024-02-13T21:45:12.469Z 28 560-579 130 130 130 241 241 241 F F F
146452024-02-13T21:45:12.469Z 29 580-599 126 126 126 234 234 234 F F F
146462024-02-13T21:45:12.469ZMax gen: 135, Max flush: 254
146472024-02-13T21:45:12.469Z
146482024-02-13T21:45:12.470ZBegin loop 68
146492024-02-13T21:45:12.470ZDownstairs to restart: 2
146502024-02-13T21:45:12.481Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.201772918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
146512024-02-13T21:45:12.481Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202082386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
146522024-02-13T21:45:12.490Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202129859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
146532024-02-13T21:45:12.490ZCrucible runtime is spawned
146542024-02-13T21:45:12.490Z{"msg":"Crucible beac4778-2514-46c0-bb53-e82437169201 has session id: c716435d-a0f7-4203-97f4-70caf1f894d8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202193086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146552024-02-13T21:45:12.490Z{"msg":"Upstairs opts: Upstairs UUID: beac4778-2514-46c0-bb53-e82437169201, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202248496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146562024-02-13T21:45:12.490Z{"msg":"Crucible stats registered with UUID: beac4778-2514-46c0-bb53-e82437169201","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.20229288Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146572024-02-13T21:45:12.491Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202333775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
146582024-02-13T21:45:12.491Z{"msg":"beac4778-2514-46c0-bb53-e82437169201 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202379868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146592024-02-13T21:45:12.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202423812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146602024-02-13T21:45:12.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202478412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146612024-02-13T21:45:12.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.20251313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146622024-02-13T21:45:12.491Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202554485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146632024-02-13T21:45:12.491Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202580146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146642024-02-13T21:45:12.492Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202603537Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146652024-02-13T21:45:12.492Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202627558Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146662024-02-13T21:45:12.492Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202668163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146672024-02-13T21:45:12.492Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202692125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146682024-02-13T21:45:12.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202721564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146692024-02-13T21:45:12.492Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202743566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146702024-02-13T21:45:12.493Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.202766728Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"io task","client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146712024-02-13T21:45:12.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.203574925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146722024-02-13T21:45:12.493Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.20361414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146732024-02-13T21:45:12.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.203632154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146742024-02-13T21:45:12.493Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.203696011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146752024-02-13T21:45:12.493Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204209484Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146762024-02-13T21:45:12.493Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204382692Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146772024-02-13T21:45:12.493Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204431724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146782024-02-13T21:45:12.494Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204474758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146792024-02-13T21:45:12.494Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204672367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146802024-02-13T21:45:12.494Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204710463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146812024-02-13T21:45:12.494Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204735884Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146822024-02-13T21:45:12.494Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204877052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146832024-02-13T21:45:12.494Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.204935261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146842024-02-13T21:45:12.494Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205019561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146852024-02-13T21:45:12.495Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205172695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146862024-02-13T21:45:12.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205582106Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146872024-02-13T21:45:12.495Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205623271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146882024-02-13T21:45:12.495Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 243, 251, 252, 248, 249, 238, 249, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205640635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146892024-02-13T21:45:12.495Z{"msg":"[0]R generation[0..12]: [130, 130, 130, 134, 134, 132, 132, 128, 132, 134, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205662177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146902024-02-13T21:45:12.495Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205678212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146912024-02-13T21:45:12.495Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 243, 251, 252, 248, 249, 238, 249, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205699334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146922024-02-13T21:45:12.495Z{"msg":"[1]R generation[0..12]: [130, 130, 130, 134, 134, 132, 132, 128, 132, 134, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205713899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146932024-02-13T21:45:12.496Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205734501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146942024-02-13T21:45:12.496Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 243, 251, 252, 248, 249, 238, 249, 251, 251, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205748826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146952024-02-13T21:45:12.496Z{"msg":"[2]R generation[0..12]: [130, 130, 130, 134, 134, 132, 132, 128, 132, 134, 134, 133]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205774477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146962024-02-13T21:45:12.496Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205788871Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682Wait for a query_work_queue command to finish before sending IO
146972024-02-13T21:45:12.496Z,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146982024-02-13T21:45:12.496Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205806095Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
146992024-02-13T21:45:12.496Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.20582115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147002024-02-13T21:45:12.497Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205834255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147012024-02-13T21:45:12.497Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.2058478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"":"downstairs","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147022024-02-13T21:45:12.497Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205861205Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147032024-02-13T21:45:12.497Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205873331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"0","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147042024-02-13T21:45:12.497Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205893824Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"1","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147052024-02-13T21:45:12.497Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205907228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"client":"2","":"downstairs","session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147062024-02-13T21:45:12.497Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205920624Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147072024-02-13T21:45:12.498Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205940137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147082024-02-13T21:45:12.498Z{"msg":"beac4778-2514-46c0-bb53-e82437169201 is now active with session: c716435d-a0f7-4203-97f4-70caf1f894d8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205953512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147092024-02-13T21:45:12.498Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.205965647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682,"session_id":"c716435d-a0f7-4203-97f4-70caf1f894d8"}
147102024-02-13T21:45:12.498Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:11.20598492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
147112024-02-13T21:45:12.498ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147122024-02-13T21:45:12.498ZLoading write count information from file "/tmp/test_repair_verify.data"
147132024-02-13T21:45:12.498ZRun Repair workload
147142024-02-13T21:45:12.498Z01/30 Write block 212 len 4096 data: 9
147152024-02-13T21:45:12.503Z02/30 Read block 118 len 24576
147162024-02-13T21:45:12.504Z03/30 Read block 99 len 4096
147172024-02-13T21:45:12.504Z04/30 Write block 496 len 8192 data: 10 10
147182024-02-13T21:45:12.504Z05/30 Read block 335 len 32768
147192024-02-13T21:45:12.504Z06/30 Read block 370 len 36864
147202024-02-13T21:45:12.504Z07/30 Read block 96 len 20480
147212024-02-13T21:45:12.504Z08/30 Read block 200 len 24576
147222024-02-13T21:45:12.505Z09/30 Write block 472 len 32768 data: 15 17 17 15 12 11 12 9
147232024-02-13T21:45:12.505Z10/30 Flush
147242024-02-13T21:45:12.505Z11/30 Write block 124 len 4096 data: 11
147252024-02-13T21:45:12.505Z12/30 Read block 318 len 28672
147262024-02-13T21:45:12.505Z13/30 Write block 161 len 20480 data: 10 10 9 7 5
147272024-02-13T21:45:12.505Z14/30 Write block 589 len 8192 data: 10 11
147282024-02-13T21:45:12.506Z15/30 Read block 3 len 36864
147292024-02-13T21:45:12.506Z16/30 Read block 59 len 4096
147302024-02-13T21:45:12.506Z17/30 Write block 216 len 20480 data: 13 15 16 17 16
147312024-02-13T21:45:12.506Z18/30 Read block 217 len 16384
147322024-02-13T21:45:12.506Z19/30 Read block 513 len 8192
147332024-02-13T21:45:12.506Z20/30 Write block 348 len 8192 data: 11 9
147342024-02-13T21:45:12.507Z21/30 Read block 105 len 36864
147352024-02-13T21:45:12.511Z22/30 Read block 114 len 4096
147362024-02-13T21:45:12.511Z23/30 Read block 304 len 40960
147372024-02-13T21:45:12.511Z24/30 Write block 526 len 12288 data: 7 9 9
147382024-02-13T21:45:12.511Z25/30 Read block 240 len 28672
147392024-02-13T21:45:12.512Z26/30 Write block 148 len 40960 data: 8 8 8 8 7 10 10 11 9 9
147402024-02-13T21:45:12.512Z27/30 Write block 445 len 4096 data: 13
147412024-02-13T21:45:12.512Z28/30 Write block 56 len 36864 data: 13 13 12 14 12 12 11 9 6
147422024-02-13T21:45:12.512Z29/30 Write block 95 len 16384 data: 8 9 10 10
147432024-02-13T21:45:12.512Z30/30 Read block 583 len 32768
147442024-02-13T21:45:12.517Z----------------------------------------------------------------
147452024-02-13T21:45:12.517Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
147462024-02-13T21:45:12.517ZStates: Active Active Active
147472024-02-13T21:45:12.518ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147482024-02-13T21:45:12.518Z 1 Acked 1000 Write 1 Done Done Done false
147492024-02-13T21:45:12.518Z 2 Acked 1001 Read 6 Done Done Sent false
147502024-02-13T21:45:12.518Z 3 Acked 1002 Read 1 Done Done Sent false
147512024-02-13T21:45:12.518Z 4 Acked 1003 Write 2 Done Done Sent false
147522024-02-13T21:45:12.518Z 5 Acked 1004 Read 8 Done Done Sent false
147532024-02-13T21:45:12.519Z 6 Acked 1005 Read 9 Done Done Sent false
147542024-02-13T21:45:12.519Z 7 Acked 1006 Read 5 Done Done Sent false
147552024-02-13T21:45:12.519Z 8 Acked 1007 Read 6 Done Done Sent false
147562024-02-13T21:45:12.519Z 9 Acked 1008 Write 8 Done Done Sent false
147572024-02-13T21:45:12.519Z 10 Acked 1009 Flush 0 Done Done Sent false
147582024-02-13T21:45:12.519Z 11 Acked 1010 Write 1 Done Done Sent false
147592024-02-13T21:45:12.520Z 12 Acked 1011 Read 7 Done Done Sent false
147602024-02-13T21:45:12.520Z 13 Acked 1012 Write 5 Done Done Sent false
147612024-02-13T21:45:12.520Z 14 Acked 1013 Write 2 Done Done Sent false
147622024-02-13T21:45:12.520Z 15 Acked 1014 Read 9 Done Done Sent false
147632024-02-13T21:45:12.520Z 16 Acked 1015 Read 1 Done Done Sent false
147642024-02-13T21:45:12.520Z 17 Acked 1016 Write 5 Done Done Sent false
147652024-02-13T21:45:12.521Z 18 Acked 1017 Read 4 Done Done Sent false
147662024-02-13T21:45:12.521Z 19 Acked 1018 Read 2 Done Done Sent false
147672024-02-13T21:45:12.521Z 20 Acked 1019 Write 2 Done Done Sent false
147682024-02-13T21:45:12.521Z 21 Acked 1020 Read 9 Done Done Sent false
147692024-02-13T21:45:12.521Z 22 Acked 1021 Read 1 Done Done Sent false
147702024-02-13T21:45:12.521Z 23 Acked 1022 Read 10 Done Done Sent false
147712024-02-13T21:45:12.521Z 24 Acked 1023 Write 3 Done Done Sent false
147722024-02-13T21:45:12.521Z 25 Acked 1024 Read 7 Done Done Sent false
147732024-02-13T21:45:12.522Z 26 Acked 1025 Write 10 Done Done Sent false
147742024-02-13T21:45:12.522Z 27 Acked 1026 Write 1 Done Done Sent false
147752024-02-13T21:45:12.522Z 28 Acked 1027 Write 9 Done Done Sent false
147762024-02-13T21:45:12.522Z 29 Acked 1028 Write 4 Done Done Sent false
147772024-02-13T21:45:12.522Z 30 Acked 1029 Read 8 Done Done Sent false
147782024-02-13T21:45:12.522Z STATES DS:0 DS:1 DS:2 TOTAL
147792024-02-13T21:45:12.522Z New 0 0 0 0
147802024-02-13T21:45:12.523Z Sent 0 0 29 29
147812024-02-13T21:45:12.523Z Done 30 30 1 61
147822024-02-13T21:45:12.523Z Skipped 0 0 0 0
147832024-02-13T21:45:12.523Z Error 0 0 0 0
147842024-02-13T21:45:12.524ZLast Flush: 1009 1009 0
147852024-02-13T21:45:12.524ZDownstairs last five completed:
147862024-02-13T21:45:12.524ZUpstairs last five completed: 30 29 28 27 26
147872024-02-13T21:45:12.524Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:11.23264368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1682}
147882024-02-13T21:45:12.524ZWrote out file "/tmp/test_repair_verify.data"
147892024-02-13T21:45:12.524Z
147902024-02-13T21:45:12.529ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147912024-02-13T21:45:12.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147922024-02-13T21:45:12.538Z 0 000-019 130 130 130 242 242 242 F F F
147932024-02-13T21:45:12.538Z 1 020-039 130 130 130 242 242 242 F F F
147942024-02-13T21:45:12.538Z 2 040-059 130 130 130 243 243 243 T T F <---
147952024-02-13T21:45:12.538Z 3 060-079 134 134 134 251 251 251 T T F <---
147962024-02-13T21:45:12.538Z 4 080-099 134 134 134 252 252 252 T T F <---
147972024-02-13T21:45:12.538Z 5 100-119 132 132 132 248 248 248 F F F
147982024-02-13T21:45:12.539Z 6 120-139 132 132 132 249 249 249 T T F <---
147992024-02-13T21:45:12.539Z 7 140-159 128 128 128 238 238 238 T T F <---
148002024-02-13T21:45:12.544Z 8 160-179 132 132 132 249 249 249 T T F <---
148012024-02-13T21:45:12.544Z 9 180-199 134 134 134 251 251 251 F F F
148022024-02-13T21:45:12.545Z 10 200-219 136 136 134 255 255 251 T T T <---
148032024-02-13T21:45:12.545Z 11 220-239 133 133 133 250 250 250 T T F <---
148042024-02-13T21:45:12.545Z 12 240-259 131 131 131 245 245 245 F F F
148052024-02-13T21:45:12.545Z 13 260-279 131 131 131 245 245 245 F F F
148062024-02-13T21:45:12.545Z 14 280-299 133 133 133 250 250 250 F F F
148072024-02-13T21:45:12.545Z 15 300-319 133 133 133 250 250 250 F F F
148082024-02-13T21:45:12.545Z 16 320-339 123 123 123 229 229 229 F F F
148092024-02-13T21:45:12.545Z 17 340-359 135 135 135 254 254 254 T T F <---
148102024-02-13T21:45:12.545Z 18 360-379 135 135 135 254 254 254 F F F
148112024-02-13T21:45:12.545Z 19 380-399 129 129 129 239 239 239 F F F
148122024-02-13T21:45:12.546Z 20 400-419 134 134 134 251 251 251 F F F
148132024-02-13T21:45:12.546Z 21 420-439 131 131 131 245 245 245 F F F
148142024-02-13T21:45:12.546Z 22 440-459 132 132 132 248 248 248 T T F <---
148152024-02-13T21:45:12.546Z 23 460-479 136 136 132 255 255 248 F F F <---
148162024-02-13T21:45:12.546Z 24 480-499 136 136 132 255 255 249 F F F <---
148172024-02-13T21:45:12.546Z 25 500-519 135 135 135 254 254 254 F F F
148182024-02-13T21:45:12.546Z 26 520-539 135 135 135 254 254 254 T T F <---
148192024-02-13T21:45:12.546Z 27 540-559 132 132 132 248 248 248 F F F
148202024-02-13T21:45:12.546Z 28 560-579 130 130 130 241 241 241 F F F
148212024-02-13T21:45:12.547Z 29 580-599 126 126 126 234 234 234 T T F <---
148222024-02-13T21:45:12.547ZMax gen: 136, Max flush: 255
148232024-02-13T21:45:12.547ZError: Difference in extent metadata found!
148242024-02-13T21:45:12.547ZA Difference in extent metadata is expected here
148252024-02-13T21:45:12.547ZOn loop 68
148262024-02-13T21:45:12.547Z
148272024-02-13T21:45:12.547ZVerifying data now
148282024-02-13T21:45:17.249ZNo change after verify
148292024-02-13T21:45:17.249ZLoop: 68 Downstairs dump after verify (and repair):
148302024-02-13T21:45:17.263ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148312024-02-13T21:45:17.264Z 0 000-019 130 130 130 242 242 242 F F F
148322024-02-13T21:45:17.264Z 1 020-039 130 130 130 242 242 242 F F F
148332024-02-13T21:45:17.264Z 2 040-059 137 137 137 256 256 256 F F F
148342024-02-13T21:45:17.264Z 3 060-079 137 137 137 256 256 256 F F F
148352024-02-13T21:45:17.264Z 4 080-099 137 137 137 256 256 256 F F F
148362024-02-13T21:45:17.264Z 5 100-119 132 132 132 248 248 248 F F F
148372024-02-13T21:45:17.264Z 6 120-139 137 137 137 256 256 256 F F F
148382024-02-13T21:45:17.265Z 7 140-159 137 137 137 256 256 256 F F F
148392024-02-13T21:45:17.265Z 8 160-179 137 137 137 256 256 256 F F F
148402024-02-13T21:45:17.265Z 9 180-199 134 134 134 251 251 251 F F F
148412024-02-13T21:45:17.265Z 10 200-219 137 137 137 256 256 256 F F F
148422024-02-13T21:45:17.265Z 11 220-239 137 137 137 256 256 256 F F F
148432024-02-13T21:45:17.270Z 12 240-259 131 131 131 245 245 245 F F F
148442024-02-13T21:45:17.270Z 13 260-279 131 131 131 245 245 245 F F F
148452024-02-13T21:45:17.270Z 14 280-299 133 133 133 250 250 250 F F F
148462024-02-13T21:45:17.271Z 15 300-319 133 133 133 250 250 250 F F F
148472024-02-13T21:45:17.271Z 16 320-339 123 123 123 229 229 229 F F F
148482024-02-13T21:45:17.271Z 17 340-359 137 137 137 256 256 256 F F F
148492024-02-13T21:45:17.271Z 18 360-379 135 135 135 254 254 254 F F F
148502024-02-13T21:45:17.271Z 19 380-399 129 129 129 239 239 239 F F F
148512024-02-13T21:45:17.272Z 20 400-419 134 134 134 251 251 251 F F F
148522024-02-13T21:45:17.272Z 21 420-439 131 131 131 245 245 245 F F F
148532024-02-13T21:45:17.272Z 22 440-459 137 137 137 256 256 256 F F F
148542024-02-13T21:45:17.272Z 23 460-479 136 136 136 255 255 255 F F F
148552024-02-13T21:45:17.272Z 24 480-499 136 136 136 255 255 255 F F F
148562024-02-13T21:45:17.272Z 25 500-519 135 135 135 254 254 254 F F F
148572024-02-13T21:45:17.272Z 26 520-539 137 137 137 256 256 256 F F F
148582024-02-13T21:45:17.272Z 27 540-559 132 132 132 248 248 248 F F F
148592024-02-13T21:45:17.272Z 28 560-579 130 130 130 241 241 241 F F F
148602024-02-13T21:45:17.273Z 29 580-599 137 137 137 256 256 256 F F F
148612024-02-13T21:45:17.273ZMax gen: 137, Max flush: 256
148622024-02-13T21:45:17.273Z
148632024-02-13T21:45:17.273ZBegin loop 69
148642024-02-13T21:45:17.273ZDownstairs to restart: 1
148652024-02-13T21:45:17.285Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.003806837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
148662024-02-13T21:45:17.285Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004159999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
148672024-02-13T21:45:17.285Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004189668Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
148682024-02-13T21:45:17.294Z{"msg":"Crucible d9421e19-723c-465f-b59a-61f193fe7989 has session id: 6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004237161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id"Crucible runtime is spawned
148692024-02-13T21:45:17.294Z:"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148702024-02-13T21:45:17.294Z{"msg":"Upstairs opts: Upstairs UUID: d9421e19-723c-465f-b59a-61f193fe7989, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004299188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148712024-02-13T21:45:17.294Z{"msg":"Crucible stats registered with UUID: d9421e19-723c-465f-b59a-61f193fe7989","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004329697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148722024-02-13T21:45:17.294Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004355928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
148732024-02-13T21:45:17.294Z{"msg":"d9421e19-723c-465f-b59a-61f193fe7989 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004509232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148742024-02-13T21:45:17.294Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004552266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148752024-02-13T21:45:17.294Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004580286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148762024-02-13T21:45:17.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004630758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148772024-02-13T21:45:17.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004666215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148782024-02-13T21:45:17.295Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004715687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148792024-02-13T21:45:17.295Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.00476378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148802024-02-13T21:45:17.295Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004868552Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148812024-02-13T21:45:17.295Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004908757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148822024-02-13T21:45:17.295Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.00495745Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148832024-02-13T21:45:17.296Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.004986349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148842024-02-13T21:45:17.296Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.00501167Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148852024-02-13T21:45:17.296Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.005045598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"io task","client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148862024-02-13T21:45:17.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.005523314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148872024-02-13T21:45:17.296Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.005560271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148882024-02-13T21:45:17.296Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.005622798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148892024-02-13T21:45:17.296Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.005669791Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148902024-02-13T21:45:17.297Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006189892Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148912024-02-13T21:45:17.297Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006242073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148922024-02-13T21:45:17.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006577742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148932024-02-13T21:45:17.297Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006618517Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148942024-02-13T21:45:17.297Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006644957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148952024-02-13T21:45:17.297Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006730636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148962024-02-13T21:45:17.297Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006752179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148972024-02-13T21:45:17.297Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006774051Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148982024-02-13T21:45:17.298Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006787975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
148992024-02-13T21:45:17.298Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.006819344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149002024-02-13T21:45:17.298Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007281976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149012024-02-13T21:45:17.298Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007678862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149022024-02-13T21:45:17.298Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007703773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149032024-02-13T21:45:17.298Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 256, 256, 256, 248, 256, 256, 256, 251, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007718998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149042024-02-13T21:45:17.298Z{"msg":"[0]R generation[0..12]: [130, 130, 137, 137, 137, 132, 137, 137, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007743619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149052024-02-13T21:45:17.299Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007759433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149062024-02-13T21:45:17.299Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 256, 256, 256, 248, 256, 256, 256, 251, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007780506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149072024-02-13T21:45:17.299Z{"msg":"[1]R generation[0..12]: [130, 130, 137, 137, 137, 132, 137, 137, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.00779482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149082024-02-13T21:45:17.299Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007815643Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149092024-02-13T21:45:17.299Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 256, 256, 256, 248, 256, 256, 256, 251, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007834906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149102024-02-13T21:45:17.299Z{"msg":"[2]R generation[0..12]: [130, 130, 137, 137, 137, 132, 137, 137, 137, 134, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007860716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149112024-02-13T21:45:17.299Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.00787899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149122024-02-13T21:45:17.299ZWait for a query_work_queue command to finish before sending IO
149132024-02-13T21:45:17.299Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007908399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149142024-02-13T21:45:17.299Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007926333Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149152024-02-13T21:45:17.299Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007950114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149162024-02-13T21:45:17.300Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007968187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"":"downstairs","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149172024-02-13T21:45:17.300Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.007993258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149182024-02-13T21:45:17.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008008623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"0","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149192024-02-13T21:45:17.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008033054Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"1","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149202024-02-13T21:45:17.300Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008050008Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"client":"2","":"downstairs","session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149212024-02-13T21:45:17.300Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008074339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149222024-02-13T21:45:17.300Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008090133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149232024-02-13T21:45:17.300Z{"msg":"d9421e19-723c-465f-b59a-61f193fe7989 is now active with session: 6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008114314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149242024-02-13T21:45:17.300Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008129399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690,"session_id":"6dfc0bcb-2497-4cd6-8fc1-b4a7c65bb53b"}
149252024-02-13T21:45:17.300Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:16.008145453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
149262024-02-13T21:45:17.300ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
149272024-02-13T21:45:17.301ZLoading write count information from file "/tmp/test_repair_verify.data"
149282024-02-13T21:45:17.301ZRun Repair workload
149292024-02-13T21:45:17.301Z01/30 Write block 215 len 16384 data: 12 14 16 17
149302024-02-13T21:45:17.314Z02/30 Write block 402 len 12288 data: 11 9 9
149312024-02-13T21:45:17.314Z03/30 Write block 497 len 8192 data: 11 11
149322024-02-13T21:45:17.315Z04/30 Flush
149332024-02-13T21:45:17.315Z05/30 Write block 407 len 12288 data: 8 10 13
149342024-02-13T21:45:17.315Z06/30 Write block 427 len 20480 data: 10 11 15 13 13
149352024-02-13T21:45:17.315Z07/30 Flush
149362024-02-13T21:45:17.315Z08/30 Flush
149372024-02-13T21:45:17.315Z09/30 Write block 224 len 32768 data: 15 13 10 12 11 9 10 10
149382024-02-13T21:45:17.316Z10/30 Flush
149392024-02-13T21:45:17.316Z11/30 Write block 560 len 8192 data: 9 9
149402024-02-13T21:45:17.316Z12/30 Write block 272 len 16384 data: 12 12 13 14
149412024-02-13T21:45:17.316Z13/30 Write block 329 len 36864 data: 11 11 15 15 13 11 15 14 11
149422024-02-13T21:45:17.316Z14/30 Flush
149432024-02-13T21:45:17.317Z15/30 Write block 362 len 16384 data: 11 13 11 12
149442024-02-13T21:45:17.317Z16/30 Write block 457 len 24576 data: 16 16 16 17 14 13
149452024-02-13T21:45:17.317Z17/30 Write block 445 len 28672 data: 14 13 14 12 15 15 13
149462024-02-13T21:45:17.317Z18/30 Read block 91 len 36864
149472024-02-13T21:45:17.317Z19/30 Read block 359 len 24576
149482024-02-13T21:45:17.317Z20/30 Write block 194 len 8192 data: 14 13
149492024-02-13T21:45:17.317Z21/30 Flush
149502024-02-13T21:45:17.321Z22/30 Write block 224 len 12288 data: 16 14 11
149512024-02-13T21:45:17.322Z23/30 Write block 328 len 24576 data: 14 12 12 16 16 14
149522024-02-13T21:45:17.322Z24/30 Write block 55 len 4096 data: 12
149532024-02-13T21:45:17.322Z25/30 Read block 578 len 4096
149542024-02-13T21:45:17.322Z26/30 Read block 371 len 16384
149552024-02-13T21:45:17.322Z27/30 Read block 340 len 24576
149562024-02-13T21:45:17.322Z28/30 Write block 467 len 40960 data: 13 10 11 12 14 16 18 18 16 13
149572024-02-13T21:45:17.322Z29/30 Read block 23 len 40960
149582024-02-13T21:45:17.328Z30/30 Read block 241 len 24576
149592024-02-13T21:45:17.328Z----------------------------------------------------------------
149602024-02-13T21:45:17.328Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:26
149612024-02-13T21:45:17.328ZStates: Active Active Active
149622024-02-13T21:45:17.328ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149632024-02-13T21:45:17.328Z 5 Acked 1004 Write 3 Done Sent Done false
149642024-02-13T21:45:17.329Z 6 Acked 1005 Write 5 Done Sent Done false
149652024-02-13T21:45:17.329Z 7 Acked 1006 Flush 0 Done Sent Done false
149662024-02-13T21:45:17.329Z 8 Acked 1007 Flush 0 Done Sent Done false
149672024-02-13T21:45:17.329Z 9 Acked 1008 Write 8 Done Sent Done false
149682024-02-13T21:45:17.329Z 10 Acked 1009 Flush 0 Done Sent Done false
149692024-02-13T21:45:17.329Z 11 Acked 1010 Write 2 Done Sent Done false
149702024-02-13T21:45:17.329Z 12 Acked 1011 Write 4 Done Sent Done false
149712024-02-13T21:45:17.330Z 13 Acked 1012 Write 9 Done Sent Done false
149722024-02-13T21:45:17.330Z 14 Acked 1013 Flush 0 Done Sent Done false
149732024-02-13T21:45:17.330Z 15 Acked 1014 Write 4 Done Sent Done false
149742024-02-13T21:45:17.330Z 16 Acked 1015 Write 6 Done Sent Done false
149752024-02-13T21:45:17.330Z 17 Acked 1016 Write 7 Done Sent Done false
149762024-02-13T21:45:17.330Z 18 Acked 1017 Read 9 Done Sent Done false
149772024-02-13T21:45:17.330Z 19 Acked 1018 Read 6 Done Sent Done false
149782024-02-13T21:45:17.330Z 20 Acked 1019 Write 2 Done Sent Done false
149792024-02-13T21:45:17.330Z 21 Acked 1020 Flush 0 Done Sent Done false
149802024-02-13T21:45:17.331Z 22 Acked 1021 Write 3 Done Sent Done false
149812024-02-13T21:45:17.331Z 23 Acked 1022 Write 6 Done Sent Done false
149822024-02-13T21:45:17.331Z 24 Acked 1023 Write 1 Done Sent Done false
149832024-02-13T21:45:17.331Z 25 Acked 1024 Read 1 Done Sent Done false
149842024-02-13T21:45:17.331Z 26 Acked 1025 Read 4 Done Sent Done false
149852024-02-13T21:45:17.331Z 27 Acked 1026 Read 6 Done Sent Done false
149862024-02-13T21:45:17.331Z 28 Acked 1027 Write 10 Done Sent Done false
149872024-02-13T21:45:17.332Z 29 Acked 1028 Read 10 Done Sent Done false
149882024-02-13T21:45:17.332Z 30 Acked 1029 Read 6 Sent Sent Done false
149892024-02-13T21:45:17.332Z STATES DS:0 DS:1 DS:2 TOTAL
149902024-02-13T21:45:17.332Z New 0 0 0 0
149912024-02-13T21:45:17.332Z Sent 1 26 0 27
149922024-02-13T21:45:17.332Z Done 25 0 26 51
149932024-02-13T21:45:17.332Z Skipped 0 0 0 0
149942024-02-13T21:45:17.332Z Error 0 0 0 0
149952024-02-13T21:45:17.332ZLast Flush: 1020 1003 1020
149962024-02-13T21:45:17.332ZDownstairs last five completed: 1003 1002 1001 1000
149972024-02-13T21:45:17.332ZUpstairs last five completed: 30 29 28 27 26
149982024-02-13T21:45:17.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:16.041794767Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1690}
149992024-02-13T21:45:17.333ZWrote out file "/tmp/test_repair_verify.data"
150002024-02-13T21:45:17.333Z
150012024-02-13T21:45:17.337ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150022024-02-13T21:45:17.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150032024-02-13T21:45:17.346Z 0 000-019 130 130 130 242 242 242 F F F
150042024-02-13T21:45:17.346Z 1 020-039 130 130 130 242 242 242 F F F
150052024-02-13T21:45:17.346Z 2 040-059 137 137 137 256 256 256 T F T <---
150062024-02-13T21:45:17.347Z 3 060-079 137 137 137 256 256 256 F F F
150072024-02-13T21:45:17.347Z 4 080-099 137 137 137 256 256 256 F F F
150082024-02-13T21:45:17.347Z 5 100-119 132 132 132 248 248 248 F F F
150092024-02-13T21:45:17.351Z 6 120-139 137 137 137 256 256 256 F F F
150102024-02-13T21:45:17.352Z 7 140-159 137 137 137 256 256 256 F F F
150112024-02-13T21:45:17.352Z 8 160-179 137 137 137 256 256 256 F F F
150122024-02-13T21:45:17.352Z 9 180-199 138 134 138 262 251 262 F F F <---
150132024-02-13T21:45:17.352Z 10 200-219 138 138 138 257 257 257 F F F
150142024-02-13T21:45:17.352Z 11 220-239 138 137 138 260 256 260 T F T <---
150152024-02-13T21:45:17.353Z 12 240-259 131 131 131 245 245 245 F F F
150162024-02-13T21:45:17.353Z 13 260-279 138 131 138 261 245 261 F F F <---
150172024-02-13T21:45:17.353Z 14 280-299 133 133 133 250 250 250 F F F
150182024-02-13T21:45:17.353Z 15 300-319 133 133 133 250 250 250 F F F
150192024-02-13T21:45:17.353Z 16 320-339 138 123 138 261 229 261 T F T <---
150202024-02-13T21:45:17.353Z 17 340-359 137 137 137 256 256 256 F F F
150212024-02-13T21:45:17.353Z 18 360-379 138 135 138 262 254 262 F F F <---
150222024-02-13T21:45:17.353Z 19 380-399 129 129 129 239 239 239 F F F
150232024-02-13T21:45:17.354Z 20 400-419 138 138 138 258 257 258 F F F <---
150242024-02-13T21:45:17.354Z 21 420-439 138 131 138 258 245 258 F F F <---
150252024-02-13T21:45:17.354Z 22 440-459 138 137 138 262 256 262 F F F <---
150262024-02-13T21:45:17.354Z 23 460-479 138 136 138 262 255 262 T F T <---
150272024-02-13T21:45:17.354Z 24 480-499 138 138 138 257 257 257 F F F
150282024-02-13T21:45:17.354Z 25 500-519 135 135 135 254 254 254 F F F
150292024-02-13T21:45:17.354Z 26 520-539 137 137 137 256 256 256 F F F
150302024-02-13T21:45:17.355Z 27 540-559 132 132 132 248 248 248 F F F
150312024-02-13T21:45:17.355Z 28 560-579 138 130 138 261 241 261 F F F <---
150322024-02-13T21:45:17.355Z 29 580-599 137 137 137 256 256 256 F F F
150332024-02-13T21:45:17.355ZMax gen: 138, Max flush: 262
150342024-02-13T21:45:17.355ZError: Difference in extent metadata found!
150352024-02-13T21:45:17.355ZA Difference in extent metadata is expected here
150362024-02-13T21:45:17.355ZOn loop 69
150372024-02-13T21:45:17.355Z
150382024-02-13T21:45:17.360ZVerifying data now
150392024-02-13T21:45:21.911ZNo change after verify
150402024-02-13T21:45:21.911ZLoop: 69 Downstairs dump after verify (and repair):
150412024-02-13T21:45:21.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150422024-02-13T21:45:21.926Z 0 000-019 130 130 130 242 242 242 F F F
150432024-02-13T21:45:21.926Z 1 020-039 130 130 130 242 242 242 F F F
150442024-02-13T21:45:21.926Z 2 040-059 139 139 139 263 263 263 F F F
150452024-02-13T21:45:21.926Z 3 060-079 137 137 137 256 256 256 F F F
150462024-02-13T21:45:21.926Z 4 080-099 137 137 137 256 256 256 F F F
150472024-02-13T21:45:21.927Z 5 100-119 132 132 132 248 248 248 F F F
150482024-02-13T21:45:21.927Z 6 120-139 137 137 137 256 256 256 F F F
150492024-02-13T21:45:21.927Z 7 140-159 137 137 137 256 256 256 F F F
150502024-02-13T21:45:21.927Z 8 160-179 137 137 137 256 256 256 F F F
150512024-02-13T21:45:21.927Z 9 180-199 138 138 138 262 262 262 F F F
150522024-02-13T21:45:21.927Z 10 200-219 138 138 138 257 257 257 F F F
150532024-02-13T21:45:21.928Z 11 220-239 139 139 139 263 263 263 F F F
150542024-02-13T21:45:21.932Z 12 240-259 131 131 131 245 245 245 F F F
150552024-02-13T21:45:21.932Z 13 260-279 138 138 138 261 261 261 F F F
150562024-02-13T21:45:21.932Z 14 280-299 133 133 133 250 250 250 F F F
150572024-02-13T21:45:21.932Z 15 300-319 133 133 133 250 250 250 F F F
150582024-02-13T21:45:21.933Z 16 320-339 139 139 139 263 263 263 F F F
150592024-02-13T21:45:21.933Z 17 340-359 137 137 137 256 256 256 F F F
150602024-02-13T21:45:21.933Z 18 360-379 138 138 138 262 262 262 F F F
150612024-02-13T21:45:21.933Z 19 380-399 129 129 129 239 239 239 F F F
150622024-02-13T21:45:21.933Z 20 400-419 138 138 138 258 258 258 F F F
150632024-02-13T21:45:21.933Z 21 420-439 138 138 138 258 258 258 F F F
150642024-02-13T21:45:21.933Z 22 440-459 138 138 138 262 262 262 F F F
150652024-02-13T21:45:21.934Z 23 460-479 139 139 139 263 263 263 F F F
150662024-02-13T21:45:21.934Z 24 480-499 138 138 138 257 257 257 F F F
150672024-02-13T21:45:21.934Z 25 500-519 135 135 135 254 254 254 F F F
150682024-02-13T21:45:21.934Z 26 520-539 137 137 137 256 256 256 F F F
150692024-02-13T21:45:21.934Z 27 540-559 132 132 132 248 248 248 F F F
150702024-02-13T21:45:21.934Z 28 560-579 138 138 138 261 261 261 F F F
150712024-02-13T21:45:21.935Z 29 580-599 137 137 137 256 256 256 F F F
150722024-02-13T21:45:21.935ZMax gen: 139, Max flush: 263
150732024-02-13T21:45:21.935Z
150742024-02-13T21:45:21.935ZBegin loop 70
150752024-02-13T21:45:21.935ZDownstairs to restart: 1
150762024-02-13T21:45:21.960Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663636778Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
150772024-02-13T21:45:21.960Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663890386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
150782024-02-13T21:45:21.960Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663907009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
150792024-02-13T21:45:21.968Z{"msg":"Crucible 47b3fb55-8d02-4300-be99-1c8556809380 has session id: 8518f405-31c8-47ae-8eac-5cc314960a1c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663924433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150802024-02-13T21:45:21.968Z{"msg":"Upstairs opts: Upstairs UUID: 47b3fb55-8d02-4300-be99-1c8556809380, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663970826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150812024-02-13T21:45:21.968Z{"msg":"Crucible stats registered with UUID: 47b3fb55-8d02-4300-be99-1c8556809380","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.663998236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150822024-02-13T21:45:21.968ZCrucible runtime is spawned
150832024-02-13T21:45:21.968Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.66417967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
150842024-02-13T21:45:21.968Z{"msg":"47b3fb55-8d02-4300-be99-1c8556809380 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664288581Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150852024-02-13T21:45:21.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664323448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150862024-02-13T21:45:21.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664343631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150872024-02-13T21:45:21.969Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664381197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150882024-02-13T21:45:21.969Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664404689Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150892024-02-13T21:45:21.969Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664427731Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150902024-02-13T21:45:21.969Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664451872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150912024-02-13T21:45:21.969Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664466526Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150922024-02-13T21:45:21.969Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664509101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150932024-02-13T21:45:21.969Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664536301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150942024-02-13T21:45:21.970Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664659687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150952024-02-13T21:45:21.970Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664708759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150962024-02-13T21:45:21.970Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.664755272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"io task","client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150972024-02-13T21:45:21.970Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665287179Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150982024-02-13T21:45:21.970Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665324185Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
150992024-02-13T21:45:21.970Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665340149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151002024-02-13T21:45:21.970Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665353915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151012024-02-13T21:45:21.971Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665848675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151022024-02-13T21:45:21.971Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.665965083Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151032024-02-13T21:45:21.971Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666163291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151042024-02-13T21:45:21.971Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666230677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151052024-02-13T21:45:21.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666313976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151062024-02-13T21:45:21.971Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666350273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151072024-02-13T21:45:21.972Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666366527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151082024-02-13T21:45:21.972Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666426156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151092024-02-13T21:45:21.972Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666460193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151102024-02-13T21:45:21.972Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666475118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151112024-02-13T21:45:21.972Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.666673426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151122024-02-13T21:45:21.972Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667078059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151132024-02-13T21:45:21.972Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667106449Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151142024-02-13T21:45:21.973Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 263, 256, 256, 248, 256, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667121363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151152024-02-13T21:45:21.973Z{"msg":"[0]R generation[0..12]: [130, 130, 139, 137, 137, 132, 137, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667135878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151162024-02-13T21:45:21.973Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667157201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151172024-02-13T21:45:21.973Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 263, 256, 256, 248, 256, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667173265Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151182024-02-13T21:45:21.973Z{"msg":"[1]R generation[0..12]: [130, 130, 139, 137, 137, 132, 137, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667194207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151192024-02-13T21:45:21.974Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667209402Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151202024-02-13T21:45:21.974Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 263, 256, 256, 248, 256, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667230784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151212024-02-13T21:45:21.974Z{"msg":"[2]R generation[0..12]: [130, 130, 139, 137, 137, 132, 137, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667244799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151222024-02-13T21:45:21.974ZWait for a query_work_queue command to finish before sending IO
151232024-02-13T21:45:21.974Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.66726862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151242024-02-13T21:45:21.974Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667282865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151252024-02-13T21:45:21.974Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667302798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151262024-02-13T21:45:21.974Z{"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667315843Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151272024-02-13T21:45:21.975Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667336086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"":"downstairs","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151282024-02-13T21:45:21.975Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667349771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151292024-02-13T21:45:21.975Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667368454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"0","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151302024-02-13T21:45:21.975Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667381919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"1","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151312024-02-13T21:45:21.975Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667401242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"client":"2","":"downstairs","session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151322024-02-13T21:45:21.976Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667414757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151332024-02-13T21:45:21.976Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.66743321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151342024-02-13T21:45:21.976Z{"msg":"47b3fb55-8d02-4300-be99-1c8556809380 is now active with session: 8518f405-31c8-47ae-8eac-5cc314960a1c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667446445Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151352024-02-13T21:45:21.976Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667464819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698,"session_id":"8518f405-31c8-47ae-8eac-5cc314960a1c"}
151362024-02-13T21:45:21.976Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:20.667478904Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
151372024-02-13T21:45:21.977ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
151382024-02-13T21:45:21.977ZLoading write count information from file "/tmp/test_repair_verify.data"
151392024-02-13T21:45:21.977ZRun Repair workload
151402024-02-13T21:45:21.977Z01/30 Write block 538 len 12288 data: 14 17 19
151412024-02-13T21:45:21.977Z02/30 Write block 99 len 8192 data: 8 9
151422024-02-13T21:45:21.977Z03/30 Read block 136 len 20480
151432024-02-13T21:45:21.978Z04/30 Read block 389 len 16384
151442024-02-13T21:45:21.978Z05/30 Read block 386 len 32768
151452024-02-13T21:45:21.978Z06/30 Write block 46 len 36864 data: 8 7 7 9 9 9 9 9 10
151462024-02-13T21:45:21.978Z07/30 Read block 479 len 40960
151472024-02-13T21:45:21.978Z08/30 Read block 168 len 12288
151482024-02-13T21:45:21.978Z09/30 Read block 161 len 32768
151492024-02-13T21:45:21.978Z10/30 Read block 129 len 40960
151502024-02-13T21:45:21.979Z11/30 Write block 518 len 4096 data: 17
151512024-02-13T21:45:21.979Z12/30 Write block 125 len 12288 data: 16 16 15
151522024-02-13T21:45:21.979Z13/30 Flush
151532024-02-13T21:45:21.984Z14/30 Write block 86 len 24576 data: 10 9 9 9 8 9
151542024-02-13T21:45:21.984Z15/30 Read block 18 len 36864
151552024-02-13T21:45:21.984Z16/30 Read block 504 len 36864
151562024-02-13T21:45:21.984Z17/30 Write block 126 len 8192 data: 17 16
151572024-02-13T21:45:21.985Z18/30 Flush
151582024-02-13T21:45:21.985Z19/30 Write block 130 len 12288 data: 10 12 12
151592024-02-13T21:45:21.985Z20/30 Flush
151602024-02-13T21:45:21.985Z21/30 Write block 404 len 24576 data: 10 9 8 9 11 14
151612024-02-13T21:45:21.985Z22/30 Read block 402 len 40960
151622024-02-13T21:45:21.985Z23/30 Read block 237 len 36864
151632024-02-13T21:45:21.985Z24/30 Flush
151642024-02-13T21:45:21.985Z25/30 Flush
151652024-02-13T21:45:21.985Z26/30 Write block 278 len 4096 data: 12
151662024-02-13T21:45:21.986Z27/30 Write block 273 len 28672 data: 13 14 15 14 15 13 13
151672024-02-13T21:45:21.986Z28/30 Read block 293 len 8192
151682024-02-13T21:45:21.986Z29/30 Read block 35 len 32768
151692024-02-13T21:45:21.986Z30/30 Read block 113 len 8192
151702024-02-13T21:45:21.986Z----------------------------------------------------------------
151712024-02-13T21:45:21.987Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
151722024-02-13T21:45:21.987ZStates: Active Active Active
151732024-02-13T21:45:21.987ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151742024-02-13T21:45:21.987Z 1 Acked 1000 Write 3 Done Done Done false
151752024-02-13T21:45:21.987Z 2 Acked 1001 Write 2 Done Done Done false
151762024-02-13T21:45:21.987Z 3 Acked 1002 Read 5 Done Done Done false
151772024-02-13T21:45:21.987Z 4 Acked 1003 Read 4 Done Done Done false
151782024-02-13T21:45:21.987Z 5 Acked 1004 Read 8 Done Done Done false
151792024-02-13T21:45:21.987Z 6 Acked 1005 Write 9 Done Done Done false
151802024-02-13T21:45:21.987Z 7 Acked 1006 Read 10 Done Done Done false
151812024-02-13T21:45:21.988Z 8 Acked 1007 Read 3 Done Done Done false
151822024-02-13T21:45:21.988Z 9 Acked 1008 Read 8 Done Done Done false
151832024-02-13T21:45:21.988Z 10 Acked 1009 Read 10 Done Done Done false
151842024-02-13T21:45:21.988Z 11 Acked 1010 Write 1 Done Done Done false
151852024-02-13T21:45:21.988Z 12 Acked 1011 Write 3 Done Done Done false
151862024-02-13T21:45:21.988Z 13 Acked 1012 Flush 0 Done Sent Done false
151872024-02-13T21:45:21.988Z 14 Acked 1013 Write 6 Done Sent Done false
151882024-02-13T21:45:21.988Z 15 Acked 1014 Read 9 Done Sent Done false
151892024-02-13T21:45:21.989Z 16 Acked 1015 Read 9 Done Sent Done false
151902024-02-13T21:45:21.989Z 17 Acked 1016 Write 2 Done Sent Done false
151912024-02-13T21:45:21.989Z 18 Acked 1017 Flush 0 Done Sent Done false
151922024-02-13T21:45:21.989Z 19 Acked 1018 Write 3 Done Sent Done false
151932024-02-13T21:45:21.989Z 20 Acked 1019 Flush 0 Done Sent Done false
151942024-02-13T21:45:21.989Z 21 Acked 1020 Write 6 Done Sent Done false
151952024-02-13T21:45:21.989Z 22 Acked 1021 Read 10 Done Sent Done false
151962024-02-13T21:45:21.990Z 23 Acked 1022 Read 9 Done Sent Done false
151972024-02-13T21:45:21.990Z 24 Acked 1023 Flush 0 Done Sent Done false
151982024-02-13T21:45:21.990Z 25 Acked 1024 Flush 0 Done Sent Done false
151992024-02-13T21:45:21.990Z 26 Acked 1025 Write 1 Done Sent Done false
152002024-02-13T21:45:21.990Z 27 Acked 1026 Write 7 Done Sent Done false
152012024-02-13T21:45:21.990Z 28 Acked 1027 Read 2 Done Sent Done false
152022024-02-13T21:45:21.990Z 29 Acked 1028 Read 8 Done Sent Done false
152032024-02-13T21:45:21.990Z 30 Acked 1029 Read 2 Done Sent Done false
152042024-02-13T21:45:21.990Z STATES DS:0 DS:1 DS:2 TOTAL
152052024-02-13T21:45:21.991Z New 0 0 0 0
152062024-02-13T21:45:21.991Z Sent 0 18 0 18
152072024-02-13T21:45:21.991Z Done 30 12 30 72
152082024-02-13T21:45:21.991Z Skipped 0 0 0 0
152092024-02-13T21:45:21.991Z Error 0 0 0 0
152102024-02-13T21:45:21.991ZLast Flush: 1024 0 1024
152112024-02-13T21:45:21.991ZDownstairs last five completed:
152122024-02-13T21:45:21.992ZUpstairs last five completed: 30 29 28 27 26
152132024-02-13T21:45:21.992Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:20.700459321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1698}
152142024-02-13T21:45:21.992ZWrote out file "/tmp/test_repair_verify.data"
152152024-02-13T21:45:21.996Z
152162024-02-13T21:45:21.996ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
152172024-02-13T21:45:22.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152182024-02-13T21:45:22.007Z 0 000-019 130 130 130 242 242 242 F F F
152192024-02-13T21:45:22.007Z 1 020-039 130 130 130 242 242 242 F F F
152202024-02-13T21:45:22.007Z 2 040-059 140 139 140 264 263 264 F T F <---
152212024-02-13T21:45:22.007Z 3 060-079 137 137 137 256 256 256 F F F
152222024-02-13T21:45:22.007Z 4 080-099 140 137 140 265 256 265 F T F <---
152232024-02-13T21:45:22.007Z 5 100-119 140 132 140 264 248 264 F T F <---
152242024-02-13T21:45:22.007Z 6 120-139 140 137 140 266 256 266 F T F <---
152252024-02-13T21:45:22.008Z 7 140-159 137 137 137 256 256 256 F F F
152262024-02-13T21:45:22.012Z 8 160-179 137 137 137 256 256 256 F F F
152272024-02-13T21:45:22.012Z 9 180-199 138 138 138 262 262 262 F F F
152282024-02-13T21:45:22.012Z 10 200-219 138 138 138 257 257 257 F F F
152292024-02-13T21:45:22.012Z 11 220-239 139 139 139 263 263 263 F F F
152302024-02-13T21:45:22.012Z 12 240-259 131 131 131 245 245 245 F F F
152312024-02-13T21:45:22.012Z 13 260-279 138 138 138 261 261 261 T F T <---
152322024-02-13T21:45:22.013Z 14 280-299 133 133 133 250 250 250 F F F
152332024-02-13T21:45:22.013Z 15 300-319 133 133 133 250 250 250 F F F
152342024-02-13T21:45:22.013Z 16 320-339 139 139 139 263 263 263 F F F
152352024-02-13T21:45:22.013Z 17 340-359 137 137 137 256 256 256 F F F
152362024-02-13T21:45:22.013Z 18 360-379 138 138 138 262 262 262 F F F
152372024-02-13T21:45:22.013Z 19 380-399 129 129 129 239 239 239 F F F
152382024-02-13T21:45:22.013Z 20 400-419 140 138 140 267 258 267 F F F <---
152392024-02-13T21:45:22.014Z 21 420-439 138 138 138 258 258 258 F F F
152402024-02-13T21:45:22.014Z 22 440-459 138 138 138 262 262 262 F F F
152412024-02-13T21:45:22.014Z 23 460-479 139 139 139 263 263 263 F F F
152422024-02-13T21:45:22.014Z 24 480-499 138 138 138 257 257 257 F F F
152432024-02-13T21:45:22.014Z 25 500-519 140 135 140 264 254 264 F T F <---
152442024-02-13T21:45:22.014Z 26 520-539 140 137 140 264 256 264 F T F <---
152452024-02-13T21:45:22.014Z 27 540-559 140 132 140 264 248 264 F T F <---
152462024-02-13T21:45:22.014Z 28 560-579 138 138 138 261 261 261 F F F
152472024-02-13T21:45:22.014Z 29 580-599 137 137 137 256 256 256 F F F
152482024-02-13T21:45:22.014ZMax gen: 140, Max flush: 267
152492024-02-13T21:45:22.015ZError: Difference in extent metadata found!
152502024-02-13T21:45:22.015ZA Difference in extent metadata is expected here
152512024-02-13T21:45:22.015ZOn loop 70
152522024-02-13T21:45:22.015Z
152532024-02-13T21:45:22.019ZVerifying data now
152542024-02-13T21:45:27.486ZNo change after verify
152552024-02-13T21:45:27.486ZLoop: 70 Downstairs dump after verify (and repair):
152562024-02-13T21:45:27.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152572024-02-13T21:45:27.500Z 0 000-019 130 130 130 242 242 242 F F F
152582024-02-13T21:45:27.500Z 1 020-039 130 130 130 242 242 242 F F F
152592024-02-13T21:45:27.501Z 2 040-059 140 140 140 264 264 264 F F F
152602024-02-13T21:45:27.501Z 3 060-079 137 137 137 256 256 256 F F F
152612024-02-13T21:45:27.501Z 4 080-099 140 140 140 265 265 265 F F F
152622024-02-13T21:45:27.501Z 5 100-119 140 140 140 264 264 264 F F F
152632024-02-13T21:45:27.501Z 6 120-139 140 140 140 266 266 266 F F F
152642024-02-13T21:45:27.501Z 7 140-159 137 137 137 256 256 256 F F F
152652024-02-13T21:45:27.501Z 8 160-179 137 137 137 256 256 256 F F F
152662024-02-13T21:45:27.502Z 9 180-199 138 138 138 262 262 262 F F F
152672024-02-13T21:45:27.502Z 10 200-219 138 138 138 257 257 257 F F F
152682024-02-13T21:45:27.502Z 11 220-239 139 139 139 263 263 263 F F F
152692024-02-13T21:45:27.502Z 12 240-259 131 131 131 245 245 245 F F F
152702024-02-13T21:45:27.502Z 13 260-279 141 141 141 268 268 268 F F F
152712024-02-13T21:45:27.502Z 14 280-299 133 133 133 250 250 250 F F F
152722024-02-13T21:45:27.507Z 15 300-319 133 133 133 250 250 250 F F F
152732024-02-13T21:45:27.507Z 16 320-339 139 139 139 263 263 263 F F F
152742024-02-13T21:45:27.507Z 17 340-359 137 137 137 256 256 256 F F F
152752024-02-13T21:45:27.508Z 18 360-379 138 138 138 262 262 262 F F F
152762024-02-13T21:45:27.508Z 19 380-399 129 129 129 239 239 239 F F F
152772024-02-13T21:45:27.508Z 20 400-419 140 140 140 267 267 267 F F F
152782024-02-13T21:45:27.508Z 21 420-439 138 138 138 258 258 258 F F F
152792024-02-13T21:45:27.508Z 22 440-459 138 138 138 262 262 262 F F F
152802024-02-13T21:45:27.508Z 23 460-479 139 139 139 263 263 263 F F F
152812024-02-13T21:45:27.508Z 24 480-499 138 138 138 257 257 257 F F F
152822024-02-13T21:45:27.508Z 25 500-519 140 140 140 264 264 264 F F F
152832024-02-13T21:45:27.509Z 26 520-539 140 140 140 264 264 264 F F F
152842024-02-13T21:45:27.509Z 27 540-559 140 140 140 264 264 264 F F F
152852024-02-13T21:45:27.509Z 28 560-579 138 138 138 261 261 261 F F F
152862024-02-13T21:45:27.509Z 29 580-599 137 137 137 256 256 256 F F F
152872024-02-13T21:45:27.509ZMax gen: 141, Max flush: 268
152882024-02-13T21:45:27.509Z
152892024-02-13T21:45:27.509ZBegin loop 71
152902024-02-13T21:45:27.510ZDownstairs to restart: 2
152912024-02-13T21:45:27.522Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237249308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
152922024-02-13T21:45:27.523Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237494509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
152932024-02-13T21:45:27.523Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237511383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
152942024-02-13T21:45:27.523Z{"msg":"Crucible fba4dbd6-ba48-49d9-a752-7b1adb7d0411 has session id: 0cf6f423-0a59-45d0-ae76-b287fceace90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237533625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
152952024-02-13T21:45:27.530Z{"msg":"Upstairs opts: Upstairs UUID: fba4dbd6-ba48-49d9-a752-7b1adb7d0411, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.2375754Z"Crucible runtime is spawned
152962024-02-13T21:45:27.531Z,"hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
152972024-02-13T21:45:27.531Z{"msg":"Crucible stats registered with UUID: fba4dbd6-ba48-49d9-a752-7b1adb7d0411","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237643795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
152982024-02-13T21:45:27.531Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237672584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
152992024-02-13T21:45:27.531Z{"msg":"fba4dbd6-ba48-49d9-a752-7b1adb7d0411 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237751076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153002024-02-13T21:45:27.531Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237775747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153012024-02-13T21:45:27.531Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237791311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153022024-02-13T21:45:27.531Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237818522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153032024-02-13T21:45:27.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237841383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153042024-02-13T21:45:27.532Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237880129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153052024-02-13T21:45:27.532Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153062024-02-13T21:45:27.532Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237948144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153072024-02-13T21:45:27.532Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.237990999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153082024-02-13T21:45:27.532Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238039901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153092024-02-13T21:45:27.532Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238144543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153102024-02-13T21:45:27.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238197864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153112024-02-13T21:45:27.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238221925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"io task","client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153122024-02-13T21:45:27.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238901549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153132024-02-13T21:45:27.533Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.238949391Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153142024-02-13T21:45:27.533Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.23898196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153152024-02-13T21:45:27.533Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.239044787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153162024-02-13T21:45:27.533Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.239548534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153172024-02-13T21:45:27.533Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.239595767Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153182024-02-13T21:45:27.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240256757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153192024-02-13T21:45:27.534Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240293953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153202024-02-13T21:45:27.534Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240311847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153212024-02-13T21:45:27.534Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240343026Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153222024-02-13T21:45:27.534Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240381142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153232024-02-13T21:45:27.534Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240402494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153242024-02-13T21:45:27.535Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240432513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153252024-02-13T21:45:27.535Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240448468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153262024-02-13T21:45:27.535Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.240871464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153272024-02-13T21:45:27.535Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241284024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153282024-02-13T21:45:27.535Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241322491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153292024-02-13T21:45:27.535Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 266, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241346862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153302024-02-13T21:45:27.535Z{"msg":"[0]R generation[0..12]: [130, 130, 140, 137, 140, 140, 140, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241376061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153312024-02-13T21:45:27.536Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241394244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153322024-02-13T21:45:27.536Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 266, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241424034Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153332024-02-13T21:45:27.536Z{"msg":"[1]R generation[0..12]: [130, 130, 140, 137, 140, 140, 140, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241444836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153342024-02-13T21:45:27.536Z{"msg"Wait for a query_work_queue command to finish before sending IO
153352024-02-13T21:45:27.536Z:"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241476075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153362024-02-13T21:45:27.537Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 266, 256, 256, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241506244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153372024-02-13T21:45:27.537Z{"msg":"[2]R generation[0..12]: [130, 130, 140, 137, 140, 140, 140, 137, 137, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.2415437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153382024-02-13T21:45:27.537Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241575959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153392024-02-13T21:45:27.537Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241605957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153402024-02-13T21:45:27.537Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241637656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153412024-02-13T21:45:27.537Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241673543Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153422024-02-13T21:45:27.537Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241703782Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"":"downstairs","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153432024-02-13T21:45:27.537Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241724185Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153442024-02-13T21:45:27.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.24176465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"0","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153452024-02-13T21:45:27.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241797198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"1","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153462024-02-13T21:45:27.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241835594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"client":"2","":"downstairs","session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153472024-02-13T21:45:27.538Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241858776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153482024-02-13T21:45:27.538Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241887555Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153492024-02-13T21:45:27.539Z{"msg":"fba4dbd6-ba48-49d9-a752-7b1adb7d0411 is now active with session: 0cf6f423-0a59-45d0-ae76-b287fceace90","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241904489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153502024-02-13T21:45:27.539Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241933239Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pidRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153512024-02-13T21:45:27.539Z":1706,"session_id":"0cf6f423-0a59-45d0-ae76-b287fceace90"}
153522024-02-13T21:45:27.539Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:26.241960379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
153532024-02-13T21:45:27.539ZLoading write count information from file "/tmp/test_repair_verify.data"
153542024-02-13T21:45:27.539ZRun Repair workload
153552024-02-13T21:45:27.545Z01/30 Write block 352 len 16384 data: 11 13 14 15
153562024-02-13T21:45:27.545Z02/30 Read block 526 len 20480
153572024-02-13T21:45:27.545Z03/30 Read block 219 len 16384
153582024-02-13T21:45:27.545Z04/30 Read block 586 len 12288
153592024-02-13T21:45:27.545Z05/30 Write block 405 len 8192 data: 10 9
153602024-02-13T21:45:27.546Z06/30 Read block 370 len 8192
153612024-02-13T21:45:27.546Z07/30 Read block 117 len 36864
153622024-02-13T21:45:27.546Z08/30 Write block 175 len 16384 data: 9 9 12 12
153632024-02-13T21:45:27.546Z09/30 Read block 229 len 8192
153642024-02-13T21:45:27.546Z10/30 Read block 104 len 28672
153652024-02-13T21:45:27.547Z11/30 Write block 127 len 36864 data: 17 12 12 11 13 13 13 12 12
153662024-02-13T21:45:27.547Z12/30 Read block 44 len 36864
153672024-02-13T21:45:27.547Z13/30 Flush
153682024-02-13T21:45:27.547Z14/30 Write block 531 len 40960 data: 10 11 11 12 12 10 12 15 18 20
153692024-02-13T21:45:27.547Z15/30 Write block 454 len 4096 data: 11
153702024-02-13T21:45:27.547Z16/30 Read block 218 len 36864
153712024-02-13T21:45:27.547Z17/30 Read block 115 len 40960
153722024-02-13T21:45:27.548Z18/30 Read block 453 len 4096
153732024-02-13T21:45:27.552Z19/30 Read block 130 len 28672
153742024-02-13T21:45:27.552Z20/30 Read block 145 len 24576
153752024-02-13T21:45:27.552Z21/30 Read block 237 len 16384
153762024-02-13T21:45:27.552Z22/30 Read block 199 len 8192
153772024-02-13T21:45:27.553Z23/30 Read block 302 len 8192
153782024-02-13T21:45:27.553Z24/30 Read block 207 len 24576
153792024-02-13T21:45:27.553Z25/30 Read block 482 len 32768
153802024-02-13T21:45:27.553Z26/30 Read block 125 len 20480
153812024-02-13T21:45:27.553Z27/30 Read block 110 len 20480
153822024-02-13T21:45:27.557Z28/30 Write block 551 len 28672 data: 16 13 12 12 11 11 10
153832024-02-13T21:45:27.558Z29/30 Write block 474 len 40960 data: 19 17 14 12 13 10 10 11 12 11
153842024-02-13T21:45:27.558Z30/30 Read block 253 len 28672
153852024-02-13T21:45:27.558Z----------------------------------------------------------------
153862024-02-13T21:45:27.558Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153872024-02-13T21:45:27.558ZStates: Active Active Active
153882024-02-13T21:45:27.558ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153892024-02-13T21:45:27.559Z 1 Acked 1000 Write 4 Done Done Done false
153902024-02-13T21:45:27.559Z 2 Acked 1001 Read 5 Done Done Done false
153912024-02-13T21:45:27.559Z 3 Acked 1002 Read 4 Done Done Sent false
153922024-02-13T21:45:27.559Z 4 Acked 1003 Read 3 Done Done Sent false
153932024-02-13T21:45:27.559Z 5 Acked 1004 Write 2 Done Done Sent false
153942024-02-13T21:45:27.559Z 6 Acked 1005 Read 2 Done Done Sent false
153952024-02-13T21:45:27.560Z 7 Acked 1006 Read 9 Done Done Sent false
153962024-02-13T21:45:27.560Z 8 Acked 1007 Write 4 Done Done Sent false
153972024-02-13T21:45:27.560Z 9 Acked 1008 Read 2 Done Done Sent false
153982024-02-13T21:45:27.560Z 10 Acked 1009 Read 7 Done Done Sent false
153992024-02-13T21:45:27.560Z 11 Acked 1010 Write 9 Done Done Sent false
154002024-02-13T21:45:27.560Z 12 Acked 1011 Read 9 Done Done Sent false
154012024-02-13T21:45:27.560Z 13 Acked 1012 Flush 0 Done Done Sent false
154022024-02-13T21:45:27.560Z 14 Acked 1013 Write 10 Done Done Sent false
154032024-02-13T21:45:27.561Z 15 Acked 1014 Write 1 Done Done Sent false
154042024-02-13T21:45:27.561Z 16 Acked 1015 Read 9 Done Done Sent false
154052024-02-13T21:45:27.561Z 17 Acked 1016 Read 10 Done Done Sent false
154062024-02-13T21:45:27.561Z 18 Acked 1017 Read 1 Done Done Sent false
154072024-02-13T21:45:27.561Z 19 Acked 1018 Read 7 Done Done Sent false
154082024-02-13T21:45:27.561Z 20 Acked 1019 Read 6 Done Done Sent false
154092024-02-13T21:45:27.562Z 21 Acked 1020 Read 4 Done Done Sent false
154102024-02-13T21:45:27.562Z 22 Acked 1021 Read 2 Done Done Sent false
154112024-02-13T21:45:27.562Z 23 Acked 1022 Read 2 Done Done Sent false
154122024-02-13T21:45:27.562Z 24 Acked 1023 Read 6 Done Done Sent false
154132024-02-13T21:45:27.562Z 25 Acked 1024 Read 8 Done Done Sent false
154142024-02-13T21:45:27.562Z 26 Acked 1025 Read 5 Done Done Sent false
154152024-02-13T21:45:27.563Z 27 Acked 1026 Read 5 Done Done Sent false
154162024-02-13T21:45:27.563Z 28 Acked 1027 Write 7 Done Done Sent false
154172024-02-13T21:45:27.563Z 29 Acked 1028 Write 10 Done Done Sent false
154182024-02-13T21:45:27.563Z 30 Acked 1029 Read 7 Done Sent Sent false
154192024-02-13T21:45:27.564Z STATES DS:0 DS:1 DS:2 TOTAL
154202024-02-13T21:45:27.564Z New 0 0 0 0
154212024-02-13T21:45:27.564Z Sent 0 1 28 29
154222024-02-13T21:45:27.564Z Done 30 29 2 61
154232024-02-13T21:45:27.564Z Skipped 0 0 0 0
154242024-02-13T21:45:27.564Z Error 0 0 0 0
154252024-02-13T21:45:27.565ZLast Flush: 1012 1012 0
154262024-02-13T21:45:27.565ZDownstairs last five completed:
154272024-02-13T21:45:27.565ZUpstairs last five completed: 30 29 28 27 26
154282024-02-13T21:45:27.565Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:26.271129279Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1706}
154292024-02-13T21:45:27.565ZWrote out file "/tmp/test_repair_verify.data"
154302024-02-13T21:45:27.570Z
154312024-02-13T21:45:27.570ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154322024-02-13T21:45:27.579ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154332024-02-13T21:45:27.580Z 0 000-019 130 130 130 242 242 242 F F F
154342024-02-13T21:45:27.580Z 1 020-039 130 130 130 242 242 242 F F F
154352024-02-13T21:45:27.580Z 2 040-059 140 140 140 264 264 264 F F F
154362024-02-13T21:45:27.580Z 3 060-079 137 137 137 256 256 256 F F F
154372024-02-13T21:45:27.580Z 4 080-099 140 140 140 265 265 265 F F F
154382024-02-13T21:45:27.580Z 5 100-119 140 140 140 264 264 264 F F F
154392024-02-13T21:45:27.580Z 6 120-139 142 142 140 269 269 266 F F F <---
154402024-02-13T21:45:27.581Z 7 140-159 137 137 137 256 256 256 F F F
154412024-02-13T21:45:27.581Z 8 160-179 142 142 137 269 269 256 F F F <---
154422024-02-13T21:45:27.581Z 9 180-199 138 138 138 262 262 262 F F F
154432024-02-13T21:45:27.581Z 10 200-219 138 138 138 257 257 257 F F F
154442024-02-13T21:45:27.585Z 11 220-239 139 139 139 263 263 263 F F F
154452024-02-13T21:45:27.585Z 12 240-259 131 131 131 245 245 245 F F F
154462024-02-13T21:45:27.585Z 13 260-279 141 141 141 268 268 268 F F F
154472024-02-13T21:45:27.585Z 14 280-299 133 133 133 250 250 250 F F F
154482024-02-13T21:45:27.585Z 15 300-319 133 133 133 250 250 250 F F F
154492024-02-13T21:45:27.586Z 16 320-339 139 139 139 263 263 263 F F F
154502024-02-13T21:45:27.586Z 17 340-359 142 142 137 269 269 256 F F T <---
154512024-02-13T21:45:27.586Z 18 360-379 138 138 138 262 262 262 F F F
154522024-02-13T21:45:27.586Z 19 380-399 129 129 129 239 239 239 F F F
154532024-02-13T21:45:27.586Z 20 400-419 142 142 140 269 269 267 F F F <---
154542024-02-13T21:45:27.586Z 21 420-439 138 138 138 258 258 258 F F F
154552024-02-13T21:45:27.586Z 22 440-459 138 138 138 262 262 262 T T F <---
154562024-02-13T21:45:27.587Z 23 460-479 139 139 139 263 263 263 T T F <---
154572024-02-13T21:45:27.587Z 24 480-499 138 138 138 257 257 257 T T F <---
154582024-02-13T21:45:27.587Z 25 500-519 140 140 140 264 264 264 F F F
154592024-02-13T21:45:27.587Z 26 520-539 140 140 140 264 264 264 T T F <---
154602024-02-13T21:45:27.587Z 27 540-559 140 140 140 264 264 264 T T F <---
154612024-02-13T21:45:27.587Z 28 560-579 138 138 138 261 261 261 F F F
154622024-02-13T21:45:27.587Z 29 580-599 137 137 137 256 256 256 F F F
154632024-02-13T21:45:27.587ZMax gen: 142, Max flush: 269
154642024-02-13T21:45:27.588ZError: Difference in extent metadata found!
154652024-02-13T21:45:27.588ZA Difference in extent metadata is expected here
154662024-02-13T21:45:27.588ZOn loop 71
154672024-02-13T21:45:27.588Z
154682024-02-13T21:45:27.588ZVerifying data now
154692024-02-13T21:45:32.070ZNo change after verify
154702024-02-13T21:45:32.070ZLoop: 71 Downstairs dump after verify (and repair):
154712024-02-13T21:45:32.084ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154722024-02-13T21:45:32.084Z 0 000-019 130 130 130 242 242 242 F F F
154732024-02-13T21:45:32.085Z 1 020-039 130 130 130 242 242 242 F F F
154742024-02-13T21:45:32.085Z 2 040-059 140 140 140 264 264 264 F F F
154752024-02-13T21:45:32.085Z 3 060-079 137 137 137 256 256 256 F F F
154762024-02-13T21:45:32.085Z 4 080-099 140 140 140 265 265 265 F F F
154772024-02-13T21:45:32.085Z 5 100-119 140 140 140 264 264 264 F F F
154782024-02-13T21:45:32.085Z 6 120-139 142 142 142 269 269 269 F F F
154792024-02-13T21:45:32.085Z 7 140-159 137 137 137 256 256 256 F F F
154802024-02-13T21:45:32.085Z 8 160-179 142 142 142 269 269 269 F F F
154812024-02-13T21:45:32.085Z 9 180-199 138 138 138 262 262 262 F F F
154822024-02-13T21:45:32.085Z 10 200-219 138 138 138 257 257 257 F F F
154832024-02-13T21:45:32.086Z 11 220-239 139 139 139 263 263 263 F F F
154842024-02-13T21:45:32.086Z 12 240-259 131 131 131 245 245 245 F F F
154852024-02-13T21:45:32.086Z 13 260-279 141 141 141 268 268 268 F F F
154862024-02-13T21:45:32.090Z 14 280-299 133 133 133 250 250 250 F F F
154872024-02-13T21:45:32.090Z 15 300-319 133 133 133 250 250 250 F F F
154882024-02-13T21:45:32.090Z 16 320-339 139 139 139 263 263 263 F F F
154892024-02-13T21:45:32.090Z 17 340-359 142 142 142 269 269 269 F F F
154902024-02-13T21:45:32.090Z 18 360-379 138 138 138 262 262 262 F F F
154912024-02-13T21:45:32.090Z 19 380-399 129 129 129 239 239 239 F F F
154922024-02-13T21:45:32.091Z 20 400-419 142 142 142 269 269 269 F F F
154932024-02-13T21:45:32.091Z 21 420-439 138 138 138 258 258 258 F F F
154942024-02-13T21:45:32.091Z 22 440-459 143 143 143 270 270 270 F F F
154952024-02-13T21:45:32.091Z 23 460-479 143 143 143 270 270 270 F F F
154962024-02-13T21:45:32.091Z 24 480-499 143 143 143 270 270 270 F F F
154972024-02-13T21:45:32.091Z 25 500-519 140 140 140 264 264 264 F F F
154982024-02-13T21:45:32.091Z 26 520-539 143 143 143 270 270 270 F F F
154992024-02-13T21:45:32.091Z 27 540-559 143 143 143 270 270 270 F F F
155002024-02-13T21:45:32.092Z 28 560-579 138 138 138 261 261 261 F F F
155012024-02-13T21:45:32.092Z 29 580-599 137 137 137 256 256 256 F F F
155022024-02-13T21:45:32.092ZMax gen: 143, Max flush: 270
155032024-02-13T21:45:32.092Z
155042024-02-13T21:45:32.092ZBegin loop 72
155052024-02-13T21:45:32.092ZDownstairs to restart: 0
155062024-02-13T21:45:32.105Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.81856459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
155072024-02-13T21:45:32.105Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
155082024-02-13T21:45:32.112Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.818905096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
155092024-02-13T21:45:32.112Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.818955108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
155102024-02-13T21:45:32.112Z{"msg":"Crucible 57148686-fc14-4673-b200-ec12d01f2e1d has session id: 8e0ccc5b-861e-468b-8cf5-dd75d37df30f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.818983208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155112024-02-13T21:45:32.113Z{"msg":"Upstairs opts: Upstairs UUID: 57148686-fc14-4673-b200-ec12d01f2e1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819023733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155122024-02-13T21:45:32.113Z{"msg":"Crucible stats registered with UUID: 57148686-fc14-4673-b200-ec12d01f2e1d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819066058Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155132024-02-13T21:45:32.113Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819105813Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
155142024-02-13T21:45:32.113Z{"msg":"57148686-fc14-4673-b200-ec12d01f2e1d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819145629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155152024-02-13T21:45:32.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819173019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155162024-02-13T21:45:32.113Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819215653Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155172024-02-13T21:45:32.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819248182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155182024-02-13T21:45:32.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819285698Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155192024-02-13T21:45:32.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819317477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155202024-02-13T21:45:32.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819458436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155212024-02-13T21:45:32.114Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819519553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155222024-02-13T21:45:32.114Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819554741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155232024-02-13T21:45:32.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819601853Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155242024-02-13T21:45:32.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819627774Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155252024-02-13T21:45:32.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819649556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155262024-02-13T21:45:32.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.819684803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"io task","client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155272024-02-13T21:45:32.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820038795Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155282024-02-13T21:45:32.115Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820101102Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155292024-02-13T21:45:32.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820142257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155302024-02-13T21:45:32.115Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820182163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155312024-02-13T21:45:32.115Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820654012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155322024-02-13T21:45:32.115Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.820695677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155332024-02-13T21:45:32.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821050988Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155342024-02-13T21:45:32.116Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821098181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155352024-02-13T21:45:32.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821242189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155362024-02-13T21:45:32.116Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821291181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155372024-02-13T21:45:32.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821327707Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155382024-02-13T21:45:32.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821343412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155392024-02-13T21:45:32.116Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821357557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155402024-02-13T21:45:32.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.821390215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155412024-02-13T21:45:32.116Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82162236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155422024-02-13T21:45:32.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822071598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155432024-02-13T21:45:32.116Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82211937Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155442024-02-13T21:45:32.117Z{"msg":"[0]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 269, 256, 269, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82214757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155452024-02-13T21:45:32.117Z{"msg":"[0]R generation[0..12]: [130, 130, 140, 137, 140, 140, 142, 137, 142, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822183447Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155462024-02-13T21:45:32.117Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822220344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155472024-02-13T21:45:32.117Z{"msg":"[1]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 269, 256, 269, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82225746Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155482024-02-13T21:45:32.117ZWait for a query_work_queue command to finish before sending IO
155492024-02-13T21:45:32.117Z{"msg":"[1]R generation[0..12]: [130, 130, 140, 137, 140, 140, 142, 137, 142, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822300045Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155502024-02-13T21:45:32.117Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82234149Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155512024-02-13T21:45:32.117Z{"msg":"[2]R flush_numbers[0..12]: [242, 242, 264, 256, 265, 264, 269, 256, 269, 262, 257, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822381535Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155522024-02-13T21:45:32.117Z{"msg":"[2]R generation[0..12]: [130, 130, 140, 137, 140, 140, 142, 137, 142, 138, 138, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822419951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155532024-02-13T21:45:32.118Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822459407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155542024-02-13T21:45:32.118Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822488046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155552024-02-13T21:45:32.118Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822503091Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155562024-02-13T21:45:32.118Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822523114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155572024-02-13T21:45:32.118Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822536869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"":"downstairs","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155582024-02-13T21:45:32.118Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822556752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155592024-02-13T21:45:32.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822569277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"0","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155602024-02-13T21:45:32.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.82258948Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"1","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155612024-02-13T21:45:32.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822603245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"client":"2","":"downstairs","session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155622024-02-13T21:45:32.119Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822622898Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155632024-02-13T21:45:32.119Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822635353Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155642024-02-13T21:45:32.119Z{"msg":"57148686-fc14-4673-b200-ec12d01f2e1d is now active with session: 8e0ccc5b-861e-468b-8cf5-dd75d37df30f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822654516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155652024-02-13T21:45:32.119Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822668201Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714,"session_id":"8e0ccc5b-861e-468b-8cf5-dd75d37df30f"}
155662024-02-13T21:45:32.119Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:30.822687264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
155672024-02-13T21:45:32.119ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155682024-02-13T21:45:32.124ZLoading write count information from file "/tmp/test_repair_verify.data"
155692024-02-13T21:45:32.124ZRun Repair workload
155702024-02-13T21:45:32.124Z01/30 Flush
155712024-02-13T21:45:32.124Z02/30 Write block 311 len 24576 data: 10 12 10 9 11 12
155722024-02-13T21:45:32.124Z03/30 Write block 69 len 32768 data: 14 13 11 13 11 12 12 12
155732024-02-13T21:45:32.124Z04/30 Read block 302 len 20480
155742024-02-13T21:45:32.124Z05/30 Read block 243 len 16384
155752024-02-13T21:45:32.124Z06/30 Read block 215 len 36864
155762024-02-13T21:45:32.125Z07/30 Write block 208 len 40960 data: 7 8 9 9 10 10 10 13 15 17
155772024-02-13T21:45:32.125Z08/30 Read block 429 len 20480
155782024-02-13T21:45:32.125Z09/30 Read block 332 len 24576
155792024-02-13T21:45:32.125Z10/30 Flush
155802024-02-13T21:45:32.125Z11/30 Write block 39 len 32768 data: 10 11 11 9 8 7 7 9
155812024-02-13T21:45:32.125Z12/30 Read block 126 len 12288
155822024-02-13T21:45:32.128Z13/30 Read block 119 len 24576
155832024-02-13T21:45:32.128Z14/30 Write block 47 len 8192 data: 8 8
155842024-02-13T21:45:32.129Z15/30 Read block 110 len 36864
155852024-02-13T21:45:32.129Z16/30 Write block 187 len 8192 data: 9 12
155862024-02-13T21:45:32.129Z17/30 Write block 491 len 12288 data: 9 9 11
155872024-02-13T21:45:32.129Z18/30 Write block 216 len 12288 data: 16 18 18
155882024-02-13T21:45:32.129Z19/30 Write block 590 len 40960 data: 12 9 7 6 7 6 7 6 4 3
155892024-02-13T21:45:32.129Z20/30 Read block 307 len 28672
155902024-02-13T21:45:32.132Z21/30 Write block 506 len 24576 data: 11 11 11 13 14 15
155912024-02-13T21:45:32.132Z22/30 Write block 142 len 12288 data: 18 15 15
155922024-02-13T21:45:32.132Z23/30 Read block 177 len 40960
155932024-02-13T21:45:32.135Z24/30 Write block 542 len 8192 data: 15 15
155942024-02-13T21:45:32.135Z25/30 Write block 130 len 40960 data: 12 14 14 14 13 13 11 10 13 11
155952024-02-13T21:45:32.135Z26/30 Write block 313 len 40960 data: 11 10 12 13 13 14 17 16 15 16
155962024-02-13T21:45:32.135Z27/30 Read block 325 len 12288
155972024-02-13T21:45:32.138Z28/30 Write block 428 len 32768 data: 12 16 14 14 15 12 13 12
155982024-02-13T21:45:32.138Z29/30 Read block 131 len 8192
155992024-02-13T21:45:32.138Z30/30 Read block 225 len 40960
156002024-02-13T21:45:32.142Z----------------------------------------------------------------
156012024-02-13T21:45:32.142Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:29
156022024-02-13T21:45:32.142ZStates: Active Active Active
156032024-02-13T21:45:32.142ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156042024-02-13T21:45:32.142Z 2 Acked 1001 Write 6 Done Done Done false
156052024-02-13T21:45:32.142Z 3 Acked 1002 Write 8 Done Done Done false
156062024-02-13T21:45:32.142Z 4 Acked 1003 Read 5 Done Done Done false
156072024-02-13T21:45:32.142Z 5 Acked 1004 Read 4 Done Done Done false
156082024-02-13T21:45:32.142Z 6 Acked 1005 Read 9 Done Done Done false
156092024-02-13T21:45:32.142Z 7 Acked 1006 Write 10 Done Done Done false
156102024-02-13T21:45:32.142Z 8 Acked 1007 Read 5 Done Done Done false
156112024-02-13T21:45:32.142Z 9 Acked 1008 Read 6 Sent Done Done false
156122024-02-13T21:45:32.142Z 10 Acked 1009 Flush 0 Sent Done Done false
156132024-02-13T21:45:32.142Z 11 Acked 1010 Write 8 Sent Done Done false
156142024-02-13T21:45:32.142Z 12 Acked 1011 Read 3 Sent Done Done false
156152024-02-13T21:45:32.142Z 13 Acked 1012 Read 6 Sent Done Done false
156162024-02-13T21:45:32.142Z 14 Acked 1013 Write 2 Sent Done Done false
156172024-02-13T21:45:32.143Z 15 Acked 1014 Read 9 Sent Done Done false
156182024-02-13T21:45:32.143Z 16 Acked 1015 Write 2 Sent Done Done false
156192024-02-13T21:45:32.143Z 17 Acked 1016 Write 3 Sent Done Done false
156202024-02-13T21:45:32.143Z 18 Acked 1017 Write 3 Sent Done Done false
156212024-02-13T21:45:32.143Z 19 Acked 1018 Write 10 Sent Done Done false
156222024-02-13T21:45:32.143Z 20 Acked 1019 Read 7 Sent Done Done false
156232024-02-13T21:45:32.143Z 21 Acked 1020 Write 6 Sent Done Done false
156242024-02-13T21:45:32.143Z 22 Acked 1021 Write 3 Sent Done Done false
156252024-02-13T21:45:32.143Z 23 Acked 1022 Read 10 Sent Done Done false
156262024-02-13T21:45:32.143Z 24 Acked 1023 Write 2 Sent Done Done false
156272024-02-13T21:45:32.143Z 25 Acked 1024 Write 10 Sent Done Done false
156282024-02-13T21:45:32.143Z 26 Acked 1025 Write 10 Sent Done Done false
156292024-02-13T21:45:32.143Z 27 Acked 1026 Read 3 Sent Done Done false
156302024-02-13T21:45:32.143Z 28 Acked 1027 Write 8 Sent Done Done false
156312024-02-13T21:45:32.143Z 29 Acked 1028 Read 2 Sent Done Done false
156322024-02-13T21:45:32.143Z 30 Acked 1029 Read 10 Sent Sent Done false
156332024-02-13T21:45:32.143Z STATES DS:0 DS:1 DS:2 TOTAL
156342024-02-13T21:45:32.143Z New 0 0 0 0
156352024-02-13T21:45:32.143Z Sent 22 1 0 23
156362024-02-13T21:45:32.143Z Done 7 28 29 64
156372024-02-13T21:45:32.143Z Skipped 0 0 0 0
156382024-02-13T21:45:32.143Z Error 0 0 0 0
156392024-02-13T21:45:32.143ZLast Flush: 1000 1009 1009
156402024-02-13T21:45:32.143ZDownstairs last five completed: 1000
156412024-02-13T21:45:32.144ZUpstairs last five completed: 30 29 28 27 26
156422024-02-13T21:45:32.144Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:30.853094724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1714}
156432024-02-13T21:45:32.144ZWrote out file "/tmp/test_repair_verify.data"
156442024-02-13T21:45:32.147Z
156452024-02-13T21:45:32.147ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156462024-02-13T21:45:32.167ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156472024-02-13T21:45:32.167Z 0 000-019 130 130 130 242 242 242 F F F
156482024-02-13T21:45:32.167Z 1 020-039 130 130 130 242 242 242 F T T <---
156492024-02-13T21:45:32.167Z 2 040-059 140 140 140 264 264 264 F T T <---
156502024-02-13T21:45:32.167Z 3 060-079 137 144 144 256 272 272 T F F <---
156512024-02-13T21:45:32.167Z 4 080-099 140 140 140 265 265 265 F F F
156522024-02-13T21:45:32.167Z 5 100-119 140 140 140 264 264 264 F F F
156532024-02-13T21:45:32.167Z 6 120-139 142 142 142 269 269 269 F T T <---
156542024-02-13T21:45:32.167Z 7 140-159 137 137 137 256 256 256 F T T <---
156552024-02-13T21:45:32.167Z 8 160-179 142 142 142 269 269 269 F F F
156562024-02-13T21:45:32.170Z 9 180-199 138 138 138 262 262 262 F T T <---
156572024-02-13T21:45:32.170Z 10 200-219 138 144 144 257 272 272 T T T <---
156582024-02-13T21:45:32.170Z 11 220-239 139 139 139 263 263 263 F F F
156592024-02-13T21:45:32.170Z 12 240-259 131 131 131 245 245 245 F F F
156602024-02-13T21:45:32.170Z 13 260-279 141 141 141 268 268 268 F F F
156612024-02-13T21:45:32.170Z 14 280-299 133 133 133 250 250 250 F F F
156622024-02-13T21:45:32.170Z 15 300-319 133 144 144 250 272 272 T T T <---
156632024-02-13T21:45:32.170Z 16 320-339 139 139 139 263 263 263 F T T <---
156642024-02-13T21:45:32.171Z 17 340-359 142 142 142 269 269 269 F F F
156652024-02-13T21:45:32.171Z 18 360-379 138 138 138 262 262 262 F F F
156662024-02-13T21:45:32.171Z 19 380-399 129 129 129 239 239 239 F F F
156672024-02-13T21:45:32.171Z 20 400-419 142 142 142 269 269 269 F F F
156682024-02-13T21:45:32.171Z 21 420-439 138 138 138 258 258 258 F T T <---
156692024-02-13T21:45:32.171Z 22 440-459 143 143 143 270 270 270 F F F
156702024-02-13T21:45:32.171Z 23 460-479 143 143 143 270 270 270 F F F
156712024-02-13T21:45:32.171Z 24 480-499 143 143 143 270 270 270 F T T <---
156722024-02-13T21:45:32.171Z 25 500-519 140 140 140 264 264 264 F T T <---
156732024-02-13T21:45:32.171Z 26 520-539 143 143 143 270 270 270 F F F
156742024-02-13T21:45:32.171Z 27 540-559 143 143 143 270 270 270 F T T <---
156752024-02-13T21:45:32.171Z 28 560-579 138 138 138 261 261 261 F F F
156762024-02-13T21:45:32.171Z 29 580-599 137 137 137 256 256 256 F T T <---
156772024-02-13T21:45:32.171ZMax gen: 144, Max flush: 272
156782024-02-13T21:45:32.171ZError: Difference in extent metadata found!
156792024-02-13T21:45:32.171ZA Difference in extent metadata is expected here
156802024-02-13T21:45:32.171ZOn loop 72
156812024-02-13T21:45:32.171Z
156822024-02-13T21:45:32.174ZVerifying data now
156832024-02-13T21:45:37.872ZNo change after verify
156842024-02-13T21:45:37.873ZLoop: 72 Downstairs dump after verify (and repair):
156852024-02-13T21:45:37.887ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156862024-02-13T21:45:37.887Z 0 000-019 130 130 130 242 242 242 F F F
156872024-02-13T21:45:37.888Z 1 020-039 145 145 145 273 273 273 F F F
156882024-02-13T21:45:37.888Z 2 040-059 145 145 145 273 273 273 F F F
156892024-02-13T21:45:37.888Z 3 060-079 144 144 144 272 272 272 F F F
156902024-02-13T21:45:37.888Z 4 080-099 140 140 140 265 265 265 F F F
156912024-02-13T21:45:37.888Z 5 100-119 140 140 140 264 264 264 F F F
156922024-02-13T21:45:37.888Z 6 120-139 145 145 145 273 273 273 F F F
156932024-02-13T21:45:37.888Z 7 140-159 145 145 145 273 273 273 F F F
156942024-02-13T21:45:37.888Z 8 160-179 142 142 142 269 269 269 F F F
156952024-02-13T21:45:37.888Z 9 180-199 145 145 145 273 273 273 F F F
156962024-02-13T21:45:37.892Z 10 200-219 145 145 145 273 273 273 F F F
156972024-02-13T21:45:37.892Z 11 220-239 139 139 139 263 263 263 F F F
156982024-02-13T21:45:37.893Z 12 240-259 131 131 131 245 245 245 F F F
156992024-02-13T21:45:37.893Z 13 260-279 141 141 141 268 268 268 F F F
157002024-02-13T21:45:37.893Z 14 280-299 133 133 133 250 250 250 F F F
157012024-02-13T21:45:37.893Z 15 300-319 145 145 145 273 273 273 F F F
157022024-02-13T21:45:37.893Z 16 320-339 145 145 145 273 273 273 F F F
157032024-02-13T21:45:37.893Z 17 340-359 142 142 142 269 269 269 F F F
157042024-02-13T21:45:37.894Z 18 360-379 138 138 138 262 262 262 F F F
157052024-02-13T21:45:37.894Z 19 380-399 129 129 129 239 239 239 F F F
157062024-02-13T21:45:37.894Z 20 400-419 142 142 142 269 269 269 F F F
157072024-02-13T21:45:37.894Z 21 420-439 145 145 145 273 273 273 F F F
157082024-02-13T21:45:37.894Z 22 440-459 143 143 143 270 270 270 F F F
157092024-02-13T21:45:37.895Z 23 460-479 143 143 143 270 270 270 F F F
157102024-02-13T21:45:37.895Z 24 480-499 145 145 145 273 273 273 F F F
157112024-02-13T21:45:37.895Z 25 500-519 145 145 145 273 273 273 F F F
157122024-02-13T21:45:37.895Z 26 520-539 143 143 143 270 270 270 F F F
157132024-02-13T21:45:37.895Z 27 540-559 145 145 145 273 273 273 F F F
157142024-02-13T21:45:37.895Z 28 560-579 138 138 138 261 261 261 F F F
157152024-02-13T21:45:37.896Z 29 580-599 145 145 145 273 273 273 F F F
157162024-02-13T21:45:37.896ZMax gen: 145, Max flush: 273
157172024-02-13T21:45:37.896Z
157182024-02-13T21:45:37.896ZBegin loop 73
157192024-02-13T21:45:37.896ZDownstairs to restart: 1
157202024-02-13T21:45:37.909Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620183562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
157212024-02-13T21:45:37.910Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620407021Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
157222024-02-13T21:45:37.910Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620431222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
157232024-02-13T21:45:37.918Z{"msg":"Crucible bb6e5bd8-9fa7-4a0e-bf98-8c2c5597e9e6 has session id: 56dfe75d-5a32-49ec-ab98-72384004b49b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620464041Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157242024-02-13T21:45:37.918Z{"msg":"Upstairs opts: Upstairs UUID: bb6e5bd8-9fa7-4a0e-bf98-8c2c5597e9e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620499728Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157252024-02-13T21:45:37.919Z{"msg":"Crucible stats registered with UUID: bb6e5bd8-9fa7-4a0e-bf98-8c2c5597e9e6","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620533636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157262024-02-13T21:45:37.919ZCrucible runtime is spawned
157272024-02-13T21:45:37.919Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620657571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
157282024-02-13T21:45:37.919Z{"msg":"bb6e5bd8-9fa7-4a0e-bf98-8c2c5597e9e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620787453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157292024-02-13T21:45:37.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620814714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157302024-02-13T21:45:37.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620830368Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157312024-02-13T21:45:37.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620844493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157322024-02-13T21:45:37.919Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620872513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157332024-02-13T21:45:37.919Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.620898303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157342024-02-13T21:45:37.920Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.62093489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157352024-02-13T21:45:37.920Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.62096214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157362024-02-13T21:45:37.920Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.62099107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157372024-02-13T21:45:37.920Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621019379Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157382024-02-13T21:45:37.920Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621053437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157392024-02-13T21:45:37.920Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621089504Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157402024-02-13T21:45:37.920Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621146013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"io task","client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157412024-02-13T21:45:37.920Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621820419Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157422024-02-13T21:45:37.920Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621855046Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157432024-02-13T21:45:37.920Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.62187097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157442024-02-13T21:45:37.921Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.621889114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157452024-02-13T21:45:37.921Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622353635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157462024-02-13T21:45:37.921Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622429198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157472024-02-13T21:45:37.921Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622815478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157482024-02-13T21:45:37.921Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622861421Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157492024-02-13T21:45:37.921Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622879605Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157502024-02-13T21:45:37.921Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622900547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157512024-02-13T21:45:37.921Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622915012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157522024-02-13T21:45:37.921Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.622935464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157532024-02-13T21:45:37.921Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623046634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157542024-02-13T21:45:37.921Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623074704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157552024-02-13T21:45:37.921Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623508566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157562024-02-13T21:45:37.922Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623917138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157572024-02-13T21:45:37.922Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623954675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157582024-02-13T21:45:37.922Z{"msg":"[0]R flush_numbers[0..12]: [242, 273, 273, 272, 265, 264, 273, 273, 269, 273, 273, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623970589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157592024-02-13T21:45:37.922Z{"msg":"[0]R generation[0..12]: [130, 145, 145, 144, 140, 140, 145, 145, 142, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.623991981Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157602024-02-13T21:45:37.922Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624015662Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157612024-02-13T21:45:37.922Z{"msg":"[1]R flush_numbers[0..12]: [242, 273, 273, 272, 265, 264, 273, 273, 269, 273, 273, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624039884Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157622024-02-13T21:45:37.922Z{"msg":"[1]R generation[0..12]: [130, 145, 145, 144, 140, 140, 145, 145, 142, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624054628Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157632024-02-13T21:45:37.950Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.62407669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157642024-02-13T21:45:37.950Z{"msg":"Wait for a query_work_queue command to finish before sending IO
157652024-02-13T21:45:37.951Z[2]R flush_numbers[0..12]: [242, 273, 273, 272, 265, 264, 273, 273, 269, 273, 273, 263]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624094344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157662024-02-13T21:45:37.951Z{"msg":"[2]R generation[0..12]: [130, 145, 145, 144, 140, 140, 145, 145, 142, 145, 145, 139]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624116576Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157672024-02-13T21:45:37.951Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624153012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157682024-02-13T21:45:37.951Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624180373Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157692024-02-13T21:45:37.951Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624216409Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157702024-02-13T21:45:37.951Z{"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624254746Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157712024-02-13T21:45:37.951Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624293932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"":"downstairs","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157722024-02-13T21:45:37.951Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624331148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157732024-02-13T21:45:37.951Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624364836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"0","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157742024-02-13T21:45:37.951Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624401162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"1","":"downstairs","session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157752024-02-13T21:45:37.951Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624438859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"client":"2","":"downstairs"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157762024-02-13T21:45:37.951Z,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157772024-02-13T21:45:37.951Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624482143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157782024-02-13T21:45:37.951Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624509743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157792024-02-13T21:45:37.952Z{"msg":"bb6e5bd8-9fa7-4a0e-bf98-8c2c5597e9e6 is now active with session: 56dfe75d-5a32-49ec-ab98-72384004b49b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624523588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157802024-02-13T21:45:37.952Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624544241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722,"session_id":"56dfe75d-5a32-49ec-ab98-72384004b49b"}
157812024-02-13T21:45:37.952Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:36.624557566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
157822024-02-13T21:45:37.952ZLoading write count information from file "/tmp/test_repair_verify.data"
157832024-02-13T21:45:37.952ZRun Repair workload
157842024-02-13T21:45:37.964Z01/30 Write block 453 len 16384 data: 11 12 13 16
157852024-02-13T21:45:37.964Z02/30 Write block 411 len 40960 data: 14 9 10 12 13 18 20 18 16 13
157862024-02-13T21:45:37.964Z03/30 Write block 389 len 32768 data: 11 9 8 8 9 10 8 8
157872024-02-13T21:45:37.964Z04/30 Read block 112 len 24576
157882024-02-13T21:45:37.964Z05/30 Flush
157892024-02-13T21:45:37.964Z06/30 Write block 400 len 16384 data: 12 11 12 10
157902024-02-13T21:45:37.964Z07/30 Write block 213 len 36864 data: 11 11 14 17 19 19 18 17 15
157912024-02-13T21:45:37.964Z08/30 Read block 96 len 40960
157922024-02-13T21:45:37.964Z09/30 Read block 88 len 20480
157932024-02-13T21:45:37.965Z10/30 Read block 470 len 32768
157942024-02-13T21:45:37.965Z11/30 Read block 562 len 40960
157952024-02-13T21:45:37.965Z12/30 Read block 309 len 28672
157962024-02-13T21:45:37.965Z13/30 Read block 491 len 32768
157972024-02-13T21:45:37.965Z14/30 Read block 333 len 12288
157982024-02-13T21:45:37.965Z15/30 Read block 323 len 12288
157992024-02-13T21:45:37.965Z16/30 Write block 556 len 36864 data: 12 11 10 7 10 10 10 10 12
158002024-02-13T21:45:37.965Z17/30 Flush
158012024-02-13T21:45:37.965Z18/30 Write block 166 len 16384 data: 7 4 3 4
158022024-02-13T21:45:37.965Z19/30 Flush
158032024-02-13T21:45:37.965Z20/30 Write block 234 len 4096 data: 10
158042024-02-13T21:45:37.965Z21/30 Flush
158052024-02-13T21:45:37.965Z22/30 Write block 197 len 36864 data: 16 16 15 13 13 10 9 7 8
158062024-02-13T21:45:37.965Z23/30 Read block 213 len 28672
158072024-02-13T21:45:37.965Z24/30 Flush
158082024-02-13T21:45:37.965Z25/30 Write block 144 len 8192 data: 16 14
158092024-02-13T21:45:37.966Z26/30 Write block 329 len 20480 data: 13 13 17 17 15
158102024-02-13T21:45:37.966Z27/30 Write block 303 len 8192 data: 10 10
158112024-02-13T21:45:37.966Z28/30 Write block 5 len 32768 data: 9 9 9 8 10 11 14 12
158122024-02-13T21:45:37.966Z29/30 Write block 58 len 40960 data: 13 15 13 13 12 10 7 8 8 9
158132024-02-13T21:45:37.966Z30/30 Read block 244 len 4096
158142024-02-13T21:45:37.966Z----------------------------------------------------------------
158152024-02-13T21:45:37.966Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:25
158162024-02-13T21:45:37.966ZStates: Active Active Active
158172024-02-13T21:45:37.966ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
158182024-02-13T21:45:37.966Z 6 Acked 1005 Write 4 Done Sent Done false
158192024-02-13T21:45:37.966Z 7 Acked 1006 Write 9 Done Sent Done false
158202024-02-13T21:45:37.966Z 8 Acked 1007 Read 10 Done Sent Done false
158212024-02-13T21:45:37.966Z 9 Acked 1008 Read 5 Done Sent Done false
158222024-02-13T21:45:37.966Z 10 Acked 1009 Read 8 Done Sent Done false
158232024-02-13T21:45:37.966Z 11 Acked 1010 Read 10 Done Sent Done false
158242024-02-13T21:45:37.967Z 12 Acked 1011 Read 7 Done Sent Done false
158252024-02-13T21:45:37.967Z 13 Acked 1012 Read 8 Done Sent Done false
158262024-02-13T21:45:37.967Z 14 Acked 1013 Read 3 Done Sent Done false
158272024-02-13T21:45:37.967Z 15 Acked 1014 Read 3 Done Sent Done false
158282024-02-13T21:45:37.967Z 16 Acked 1015 Write 9 Done Sent Done false
158292024-02-13T21:45:37.967Z 17 Acked 1016 Flush 0 Done Sent Done false
158302024-02-13T21:45:37.967Z 18 Acked 1017 Write 4 Done Sent Done false
158312024-02-13T21:45:37.967Z 19 Acked 1018 Flush 0 Done Sent Done false
158322024-02-13T21:45:37.968Z 20 Acked 1019 Write 1 Done Sent Done false
158332024-02-13T21:45:37.968Z 21 Acked 1020 Flush 0 Done Sent Done false
158342024-02-13T21:45:37.968Z 22 Acked 1021 Write 9 Done Sent Done false
158352024-02-13T21:45:37.968Z 23 Acked 1022 Read 7 Done Sent Done false
158362024-02-13T21:45:37.968Z 24 Acked 1023 Flush 0 Done Sent Done false
158372024-02-13T21:45:37.968Z 25 Acked 1024 Write 2 Done Sent Done false
158382024-02-13T21:45:37.968Z 26 Acked 1025 Write 5 Done Sent Done false
158392024-02-13T21:45:37.968Z 27 Acked 1026 Write 2 Done Sent Done false
158402024-02-13T21:45:37.968Z 28 Acked 1027 Write 8 Done Sent Done false
158412024-02-13T21:45:37.968Z 29 Acked 1028 Write 10 Done Sent Done false
158422024-02-13T21:45:37.968Z 30 Acked 1029 Read 1 Sent Sent Done false
158432024-02-13T21:45:37.969Z STATES DS:0 DS:1 DS:2 TOTAL
158442024-02-13T21:45:37.969Z New 0 0 0 0
158452024-02-13T21:45:37.969Z Sent 1 25 0 26
158462024-02-13T21:45:37.969Z Done 24 0 25 49
158472024-02-13T21:45:37.969Z Skipped 0 0 0 0
158482024-02-13T21:45:37.969Z Error 0 0 0 0
158492024-02-13T21:45:37.969ZLast Flush: 1023 1004 1023
158502024-02-13T21:45:37.969ZDownstairs last five completed: 1004 1003 1002 1001 1000
158512024-02-13T21:45:37.969ZUpstairs last five completed: 30 29 28 27 26
158522024-02-13T21:45:37.969Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:36.660106321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1722}
158532024-02-13T21:45:37.969ZWrote out file "/tmp/test_repair_verify.data"
158542024-02-13T21:45:37.969Z
158552024-02-13T21:45:37.969ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158562024-02-13T21:45:37.973ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158572024-02-13T21:45:37.973Z 0 000-019 130 130 130 242 242 242 T F T <---
158582024-02-13T21:45:37.973Z 1 020-039 145 145 145 273 273 273 F F F
158592024-02-13T21:45:37.973Z 2 040-059 145 145 145 273 273 273 T F T <---
158602024-02-13T21:45:37.974Z 3 060-079 144 144 144 272 272 272 T F T <---
158612024-02-13T21:45:37.974Z 4 080-099 140 140 140 265 265 265 F F F
158622024-02-13T21:45:37.974Z 5 100-119 140 140 140 264 264 264 F F F
158632024-02-13T21:45:37.974Z 6 120-139 145 145 145 273 273 273 F F F
158642024-02-13T21:45:37.974Z 7 140-159 145 145 145 273 273 273 T F T <---
158652024-02-13T21:45:37.974Z 8 160-179 146 142 146 276 269 276 F F F <---
158662024-02-13T21:45:37.978Z 9 180-199 146 145 146 278 273 278 F F F <---
158672024-02-13T21:45:37.978Z 10 200-219 146 145 146 278 273 278 F F F <---
158682024-02-13T21:45:37.978Z 11 220-239 146 139 146 277 263 277 F F F <---
158692024-02-13T21:45:37.978Z 12 240-259 131 131 131 245 245 245 F F F
158702024-02-13T21:45:37.979Z 13 260-279 141 141 141 268 268 268 F F F
158712024-02-13T21:45:37.979Z 14 280-299 133 133 133 250 250 250 F F F
158722024-02-13T21:45:37.979Z 15 300-319 145 145 145 273 273 273 T F T <---
158732024-02-13T21:45:37.979Z 16 320-339 145 145 145 273 273 273 T F T <---
158742024-02-13T21:45:37.979Z 17 340-359 142 142 142 269 269 269 F F F
158752024-02-13T21:45:37.979Z 18 360-379 138 138 138 262 262 262 F F F
158762024-02-13T21:45:37.979Z 19 380-399 146 146 146 274 274 274 F F F
158772024-02-13T21:45:37.979Z 20 400-419 146 146 146 275 274 275 F F F <---
158782024-02-13T21:45:37.979Z 21 420-439 146 146 146 274 274 274 F F F
158792024-02-13T21:45:37.979Z 22 440-459 146 146 146 274 274 274 F F F
158802024-02-13T21:45:37.979Z 23 460-479 143 143 143 270 270 270 F F F
158812024-02-13T21:45:37.979Z 24 480-499 145 145 145 273 273 273 F F F
158822024-02-13T21:45:37.979Z 25 500-519 145 145 145 273 273 273 F F F
158832024-02-13T21:45:37.979Z 26 520-539 143 143 143 270 270 270 F F F
158842024-02-13T21:45:37.980Z 27 540-559 146 145 146 275 273 275 F F F <---
158852024-02-13T21:45:37.980Z 28 560-579 146 138 146 275 261 275 F F F <---
158862024-02-13T21:45:37.980Z 29 580-599 145 145 145 273 273 273 F F F
158872024-02-13T21:45:37.980ZMax gen: 146, Max flush: 278
158882024-02-13T21:45:37.980ZError: Difference in extent metadata found!
158892024-02-13T21:45:37.980ZA Difference in extent metadata is expected here
158902024-02-13T21:45:37.980ZOn loop 73
158912024-02-13T21:45:37.980Z
158922024-02-13T21:45:37.984ZVerifying data now
158932024-02-13T21:45:42.619ZNo change after verify
158942024-02-13T21:45:42.619ZLoop: 73 Downstairs dump after verify (and repair):
158952024-02-13T21:45:42.631ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158962024-02-13T21:45:42.631Z 0 000-019 147 147 147 279 279 279 F F F
158972024-02-13T21:45:42.631Z 1 020-039 145 145 145 273 273 273 F F F
158982024-02-13T21:45:42.631Z 2 040-059 147 147 147 279 279 279 F F F
158992024-02-13T21:45:42.631Z 3 060-079 147 147 147 279 279 279 F F F
159002024-02-13T21:45:42.632Z 4 080-099 140 140 140 265 265 265 F F F
159012024-02-13T21:45:42.632Z 5 100-119 140 140 140 264 264 264 F F F
159022024-02-13T21:45:42.637Z 6 120-139 145 145 145 273 273 273 F F F
159032024-02-13T21:45:42.637Z 7 140-159 147 147 147 279 279 279 F F F
159042024-02-13T21:45:42.637Z 8 160-179 146 146 146 276 276 276 F F F
159052024-02-13T21:45:42.637Z 9 180-199 146 146 146 278 278 278 F F F
159062024-02-13T21:45:42.637Z 10 200-219 146 146 146 278 278 278 F F F
159072024-02-13T21:45:42.638Z 11 220-239 146 146 146 277 277 277 F F F
159082024-02-13T21:45:42.638Z 12 240-259 131 131 131 245 245 245 F F F
159092024-02-13T21:45:42.638Z 13 260-279 141 141 141 268 268 268 F F F
159102024-02-13T21:45:42.638Z 14 280-299 133 133 133 250 250 250 F F F
159112024-02-13T21:45:42.638Z 15 300-319 147 147 147 279 279 279 F F F
159122024-02-13T21:45:42.639Z 16 320-339 147 147 147 279 279 279 F F F
159132024-02-13T21:45:42.639Z 17 340-359 142 142 142 269 269 269 F F F
159142024-02-13T21:45:42.639Z 18 360-379 138 138 138 262 262 262 F F F
159152024-02-13T21:45:42.639Z 19 380-399 146 146 146 274 274 274 F F F
159162024-02-13T21:45:42.639Z 20 400-419 146 146 146 275 275 275 F F F
159172024-02-13T21:45:42.639Z 21 420-439 146 146 146 274 274 274 F F F
159182024-02-13T21:45:42.639Z 22 440-459 146 146 146 274 274 274 F F F
159192024-02-13T21:45:42.640Z 23 460-479 143 143 143 270 270 270 F F F
159202024-02-13T21:45:42.640Z 24 480-499 145 145 145 273 273 273 F F F
159212024-02-13T21:45:42.640Z 25 500-519 145 145 145 273 273 273 F F F
159222024-02-13T21:45:42.640Z 26 520-539 143 143 143 270 270 270 F F F
159232024-02-13T21:45:42.640Z 27 540-559 146 146 146 275 275 275 F F F
159242024-02-13T21:45:42.641Z 28 560-579 146 146 146 275 275 275 F F F
159252024-02-13T21:45:42.641Z 29 580-599 145 145 145 273 273 273 F F F
159262024-02-13T21:45:42.641ZMax gen: 147, Max flush: 279
159272024-02-13T21:45:42.641Z
159282024-02-13T21:45:42.641ZBegin loop 74
159292024-02-13T21:45:42.641ZDownstairs to restart: 1
159302024-02-13T21:45:42.653Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.361887496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
159312024-02-13T21:45:42.653ZCrucible runtime is spawned
159322024-02-13T21:45:42.662Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362193685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
159332024-02-13T21:45:42.663Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362249155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
159342024-02-13T21:45:42.663Z{"msg":"Crucible e9aeb41b-6371-4f20-b609-e3b524ad6cfa has session id: 14cd8283-8f36-4d25-a286-54308ae86430","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362294358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159352024-02-13T21:45:42.663Z{"msg":"Upstairs opts: Upstairs UUID: e9aeb41b-6371-4f20-b609-e3b524ad6cfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362347709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159362024-02-13T21:45:42.663Z{"msg":"Crucible stats registered with UUID: e9aeb41b-6371-4f20-b609-e3b524ad6cfa","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362368361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159372024-02-13T21:45:42.663Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362393672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
159382024-02-13T21:45:42.663Z{"msg":"e9aeb41b-6371-4f20-b609-e3b524ad6cfa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362408527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159392024-02-13T21:45:42.663Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362424301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159402024-02-13T21:45:42.663Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362449982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159412024-02-13T21:45:42.663Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362467295Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159422024-02-13T21:45:42.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362492286Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159432024-02-13T21:45:42.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362530712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159442024-02-13T21:45:42.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362575226Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159452024-02-13T21:45:42.664Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362613442Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159462024-02-13T21:45:42.664Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362661285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159472024-02-13T21:45:42.664Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362704659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159482024-02-13T21:45:42.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362807382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159492024-02-13T21:45:42.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.362873068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159502024-02-13T21:45:42.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.363022384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"io task","client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159512024-02-13T21:45:42.664Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.36352979Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159522024-02-13T21:45:42.664Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.363576053Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159532024-02-13T21:45:42.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.363606962Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159542024-02-13T21:45:42.665Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.363646198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159552024-02-13T21:45:42.665Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364126704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159562024-02-13T21:45:42.665Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364263464Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159572024-02-13T21:45:42.665Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364634129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159582024-02-13T21:45:42.665Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364677144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159592024-02-13T21:45:42.665Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364693528Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159602024-02-13T21:45:42.665Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364713841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159612024-02-13T21:45:42.666Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364727626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159622024-02-13T21:45:42.666Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364740951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159632024-02-13T21:45:42.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364760444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159642024-02-13T21:45:42.666Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.364773769Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159652024-02-13T21:45:42.666Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365291321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159662024-02-13T21:45:42.666Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365696124Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159672024-02-13T21:45:42.666Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365742797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159682024-02-13T21:45:42.666Z{"msg":"[0]R flush_numbers[0..12]: [279, 273, 279, 279, 265, 264, 273, 279, 276, 278, 278, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365786262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159692024-02-13T21:45:42.666Z{"msg":"[0]R generation[0..12]: [147, 145, 147, 147, 140, 140, 145, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365823198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159702024-02-13T21:45:42.666Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365857636Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159712024-02-13T21:45:42.667Z{"msg":"[1]R flush_numbers[0..12]: [279, 273, 279, 279, 265, 264, 273, 279, 276, 278, 278, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365894952Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159722024-02-13T21:45:42.667ZWait for a query_work_queue command to finish before sending IO
159732024-02-13T21:45:42.667Z{"msg":"[1]R generation[0..12]: [147, 145, 147, 147, 140, 140, 145, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365935737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159742024-02-13T21:45:42.667Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.365977922Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159752024-02-13T21:45:42.667Z{"msg":"[2]R flush_numbers[0..12]: [279, 273, 279, 279, 265, 264, 273, 279, 276, 278, 278, 277]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366035341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159762024-02-13T21:45:42.667Z{"msg":"[2]R generation[0..12]: [147, 145, 147, 147, 140, 140, 145, 147, 146, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366083834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159772024-02-13T21:45:42.667Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366125868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159782024-02-13T21:45:42.667Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366163605Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159792024-02-13T21:45:42.667Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366201221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159802024-02-13T21:45:42.668ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159812024-02-13T21:45:42.668Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366239897Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159822024-02-13T21:45:42.668Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366271396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"":"downstairs","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159832024-02-13T21:45:42.668Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.36628626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159842024-02-13T21:45:42.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366306013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"0","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159852024-02-13T21:45:42.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366320998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"1","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159862024-02-13T21:45:42.668Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.3663414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"client":"2","":"downstairs","session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159872024-02-13T21:45:42.668Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366354975Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159882024-02-13T21:45:42.668Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366374198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159892024-02-13T21:45:42.668Z{"msg":"e9aeb41b-6371-4f20-b609-e3b524ad6cfa is now active with session: 14cd8283-8f36-4d25-a286-54308ae86430","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366387664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159902024-02-13T21:45:42.669Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366406437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730,"session_id":"14cd8283-8f36-4d25-a286-54308ae86430"}
159912024-02-13T21:45:42.669Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:41.366419002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
159922024-02-13T21:45:42.669ZLoading write count information from file "/tmp/test_repair_verify.data"
159932024-02-13T21:45:42.669ZRun Repair workload
159942024-02-13T21:45:42.675Z01/30 Write block 147 len 12288 data: 10 9 9
159952024-02-13T21:45:42.675Z02/30 Read block 523 len 16384
159962024-02-13T21:45:42.675Z03/30 Read block 286 len 4096
159972024-02-13T21:45:42.675Z04/30 Read block 100 len 28672
159982024-02-13T21:45:42.675Z05/30 Write block 22 len 24576 data: 13 11 11 11 9 7
159992024-02-13T21:45:42.676Z06/30 Write block 319 len 20480 data: 18 17 16 17 18
160002024-02-13T21:45:42.676Z07/30 Write block 200 len 16384 data: 14 14 11 10
160012024-02-13T21:45:42.676Z08/30 Read block 516 len 28672
160022024-02-13T21:45:42.676Z09/30 Write block 516 len 40960 data: 15 18 18 20 22 19 16 14 10 7
160032024-02-13T21:45:42.676Z10/30 Read block 185 len 12288
160042024-02-13T21:45:42.676Z11/30 Write block 488 len 8192 data: 10 10
160052024-02-13T21:45:42.676Z12/30 Write block 338 len 8192 data: 10 9
160062024-02-13T21:45:42.676Z13/30 Read block 251 len 20480
160072024-02-13T21:45:42.677Z14/30 Write block 442 len 4096 data: 11
160082024-02-13T21:45:42.677Z15/30 Write block 308 len 24576 data: 8 6 7 11 13 12
160092024-02-13T21:45:42.677Z16/30 Read block 89 len 28672
160102024-02-13T21:45:42.677Z17/30 Read block 397 len 24576
160112024-02-13T21:45:42.677Z18/30 Read block 580 len 16384
160122024-02-13T21:45:42.677Z19/30 Read block 115 len 36864
160132024-02-13T21:45:42.677Z20/30 Write block 208 len 8192 data: 8 9
160142024-02-13T21:45:42.677Z21/30 Read block 349 len 4096
160152024-02-13T21:45:42.677Z22/30 Read block 296 len 20480
160162024-02-13T21:45:42.682Z23/30 Write block 55 len 28672 data: 13 14 14 14 16 14 14
160172024-02-13T21:45:42.683Z24/30 Flush
160182024-02-13T21:45:42.683Z25/30 Write block 235 len 32768 data: 9 7 7 11 11 12 12 13
160192024-02-13T21:45:42.683Z26/30 Write block 340 len 8192 data: 12 12
160202024-02-13T21:45:42.683Z27/30 Read block 593 len 24576
160212024-02-13T21:45:42.700Z28/30 Read block 521 len 20480
160222024-02-13T21:45:42.701Z29/30 Write block 583 len 28672 data: 8 9 10 10 10 9 11
160232024-02-13T21:45:42.701Z30/30 Write block 460 len 24576 data: 18 15 14 12 13 14
160242024-02-13T21:45:42.701Z----------------------------------------------------------------
160252024-02-13T21:45:42.701Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
160262024-02-13T21:45:42.701ZStates: Active Active Active
160272024-02-13T21:45:42.701ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160282024-02-13T21:45:42.701Z 1 Acked 1000 Write 3 Done Done Done false
160292024-02-13T21:45:42.701Z 2 Acked 1001 Read 4 Done Done Done false
160302024-02-13T21:45:42.701Z 3 Acked 1002 Read 1 Done Done Done false
160312024-02-13T21:45:42.701Z 4 Acked 1003 Read 7 Done Done Done false
160322024-02-13T21:45:42.701Z 5 Acked 1004 Write 6 Done Done Done false
160332024-02-13T21:45:42.702Z 6 Acked 1005 Write 5 Done Sent Done false
160342024-02-13T21:45:42.702Z 7 Acked 1006 Write 4 Done Sent Done false
160352024-02-13T21:45:42.702Z 8 Acked 1007 Read 7 Done Sent Done false
160362024-02-13T21:45:42.702Z 9 Acked 1008 Write 10 Done Sent Done false
160372024-02-13T21:45:42.702Z 10 Acked 1009 Read 3 Done Sent Done false
160382024-02-13T21:45:42.702Z 11 Acked 1010 Write 2 Done Sent Done false
160392024-02-13T21:45:42.702Z 12 Acked 1011 Write 2 Done Sent Done false
160402024-02-13T21:45:42.702Z 13 Acked 1012 Read 5 Done Sent Done false
160412024-02-13T21:45:42.702Z 14 Acked 1013 Write 1 Done Sent Done false
160422024-02-13T21:45:42.702Z 15 Acked 1014 Write 6 Done Sent Done false
160432024-02-13T21:45:42.702Z 16 Acked 1015 Read 7 Done Sent Done false
160442024-02-13T21:45:42.702Z 17 Acked 1016 Read 6 Done Sent Done false
160452024-02-13T21:45:42.702Z 18 Acked 1017 Read 4 Done Sent Done false
160462024-02-13T21:45:42.703Z 19 Acked 1018 Read 9 Done Sent Done false
160472024-02-13T21:45:42.703Z 20 Acked 1019 Write 2 Done Sent Done false
160482024-02-13T21:45:42.703Z 21 Acked 1020 Read 1 Done Sent Done false
160492024-02-13T21:45:42.703Z 22 Acked 1021 Read 5 Done Sent Done false
160502024-02-13T21:45:42.703Z 23 Acked 1022 Write 7 Done Sent Done false
160512024-02-13T21:45:42.703Z 24 Acked 1023 Flush 0 Done Sent Done false
160522024-02-13T21:45:42.703Z 25 Acked 1024 Write 8 Done Sent Done false
160532024-02-13T21:45:42.703Z 26 Acked 1025 Write 2 Done Sent Done false
160542024-02-13T21:45:42.703Z 27 Acked 1026 Read 6 Done Sent Done false
160552024-02-13T21:45:42.704Z 28 Acked 1027 Read 5 Done Sent Done false
160562024-02-13T21:45:42.704Z 29 Acked 1028 Write 7 Done Sent Sent false
160572024-02-13T21:45:42.704Z 30 Acked 1029 Write 6 Sent Sent Sent false
160582024-02-13T21:45:42.705Z STATES DS:0 DS:1 DS:2 TOTAL
160592024-02-13T21:45:42.705Z New 0 0 0 0
160602024-02-13T21:45:42.705Z Sent 1 25 2 28
160612024-02-13T21:45:42.705Z Done 29 5 28 62
160622024-02-13T21:45:42.705Z Skipped 0 0 0 0
160632024-02-13T21:45:42.705Z Error 0 0 0 0
160642024-02-13T21:45:42.705ZLast Flush: 1023 0 1023
160652024-02-13T21:45:42.705ZDownstairs last five completed:
160662024-02-13T21:45:42.705ZUpstairs last five completed: 30 29 28 27 26
160672024-02-13T21:45:42.706Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:41.395560881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1730}
160682024-02-13T21:45:42.706ZWrote out file "/tmp/test_repair_verify.data"
160692024-02-13T21:45:42.711Z
160702024-02-13T21:45:42.711ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160712024-02-13T21:45:42.711ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160722024-02-13T21:45:42.711Z 0 000-019 147 147 147 279 279 279 F F F
160732024-02-13T21:45:42.711Z 1 020-039 148 145 148 280 273 280 F T F <---
160742024-02-13T21:45:42.712Z 2 040-059 148 147 148 280 279 280 F F F <---
160752024-02-13T21:45:42.712Z 3 060-079 148 147 148 280 279 280 F F F <---
160762024-02-13T21:45:42.712Z 4 080-099 140 140 140 265 265 265 F F F
160772024-02-13T21:45:42.712Z 5 100-119 140 140 140 264 264 264 F F F
160782024-02-13T21:45:42.712Z 6 120-139 145 145 145 273 273 273 F F F
160792024-02-13T21:45:42.712Z 7 140-159 148 147 148 280 279 280 F T F <---
160802024-02-13T21:45:42.713Z 8 160-179 146 146 146 276 276 276 F F F
160812024-02-13T21:45:42.713Z 9 180-199 146 146 146 278 278 278 F F F
160822024-02-13T21:45:42.713Z 10 200-219 148 146 148 280 278 280 F F F <---
160832024-02-13T21:45:42.713Z 11 220-239 146 146 146 277 277 277 T F T <---
160842024-02-13T21:45:42.713Z 12 240-259 131 131 131 245 245 245 T F T <---
160852024-02-13T21:45:42.713Z 13 260-279 141 141 141 268 268 268 F F F
160862024-02-13T21:45:42.713Z 14 280-299 133 133 133 250 250 250 F F F
160872024-02-13T21:45:42.713Z 15 300-319 148 147 148 280 279 280 F F F <---
160882024-02-13T21:45:42.713Z 16 320-339 148 147 148 280 279 280 F F F <---
160892024-02-13T21:45:42.713Z 17 340-359 142 142 142 269 269 269 T F T <---
160902024-02-13T21:45:42.714Z 18 360-379 138 138 138 262 262 262 F F F
160912024-02-13T21:45:42.714Z 19 380-399 146 146 146 274 274 274 F F F
160922024-02-13T21:45:42.714Z 20 400-419 146 146 146 275 275 275 F F F
160932024-02-13T21:45:42.714Z 21 420-439 146 146 146 274 274 274 F F F
160942024-02-13T21:45:42.714Z 22 440-459 148 146 148 280 274 280 F F F <---
160952024-02-13T21:45:42.714Z 23 460-479 143 143 143 270 270 270 T F T <---
160962024-02-13T21:45:42.714Z 24 480-499 148 145 148 280 273 280 F F F <---
160972024-02-13T21:45:42.714Z 25 500-519 148 145 148 280 273 280 F F F <---
160982024-02-13T21:45:42.714Z 26 520-539 148 143 148 280 270 280 F F F <---
160992024-02-13T21:45:42.714Z 27 540-559 146 146 146 275 275 275 F F F
161002024-02-13T21:45:42.714Z 28 560-579 146 146 146 275 275 275 F F F
161012024-02-13T21:45:42.715Z 29 580-599 145 145 145 273 273 273 T F T <---
161022024-02-13T21:45:42.715ZMax gen: 148, Max flush: 280
161032024-02-13T21:45:42.715ZError: Difference in extent metadata found!
161042024-02-13T21:45:42.719ZA Difference in extent metadata is expected here
161052024-02-13T21:45:42.720ZOn loop 74
161062024-02-13T21:45:42.720Z
161072024-02-13T21:45:42.720ZVerifying data now
161082024-02-13T21:45:48.472ZNo change after verify
161092024-02-13T21:45:48.472ZLoop: 74 Downstairs dump after verify (and repair):
161102024-02-13T21:45:48.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161112024-02-13T21:45:48.487Z 0 000-019 147 147 147 279 279 279 F F F
161122024-02-13T21:45:48.487Z 1 020-039 148 148 148 280 280 280 F F F
161132024-02-13T21:45:48.487Z 2 040-059 148 148 148 280 280 280 F F F
161142024-02-13T21:45:48.487Z 3 060-079 148 148 148 280 280 280 F F F
161152024-02-13T21:45:48.487Z 4 080-099 140 140 140 265 265 265 F F F
161162024-02-13T21:45:48.488Z 5 100-119 140 140 140 264 264 264 F F F
161172024-02-13T21:45:48.488Z 6 120-139 145 145 145 273 273 273 F F F
161182024-02-13T21:45:48.488Z 7 140-159 148 148 148 280 280 280 F F F
161192024-02-13T21:45:48.488Z 8 160-179 146 146 146 276 276 276 F F F
161202024-02-13T21:45:48.488Z 9 180-199 146 146 146 278 278 278 F F F
161212024-02-13T21:45:48.492Z 10 200-219 148 148 148 280 280 280 F F F
161222024-02-13T21:45:48.492Z 11 220-239 149 149 149 281 281 281 F F F
161232024-02-13T21:45:48.493Z 12 240-259 149 149 149 281 281 281 F F F
161242024-02-13T21:45:48.493Z 13 260-279 141 141 141 268 268 268 F F F
161252024-02-13T21:45:48.493Z 14 280-299 133 133 133 250 250 250 F F F
161262024-02-13T21:45:48.493Z 15 300-319 148 148 148 280 280 280 F F F
161272024-02-13T21:45:48.493Z 16 320-339 148 148 148 280 280 280 F F F
161282024-02-13T21:45:48.493Z 17 340-359 149 149 149 281 281 281 F F F
161292024-02-13T21:45:48.493Z 18 360-379 138 138 138 262 262 262 F F F
161302024-02-13T21:45:48.493Z 19 380-399 146 146 146 274 274 274 F F F
161312024-02-13T21:45:48.493Z 20 400-419 146 146 146 275 275 275 F F F
161322024-02-13T21:45:48.494Z 21 420-439 146 146 146 274 274 274 F F F
161332024-02-13T21:45:48.494Z 22 440-459 148 148 148 280 280 280 F F F
161342024-02-13T21:45:48.494Z 23 460-479 149 149 149 281 281 281 F F F
161352024-02-13T21:45:48.494Z 24 480-499 148 148 148 280 280 280 F F F
161362024-02-13T21:45:48.494Z 25 500-519 148 148 148 280 280 280 F F F
161372024-02-13T21:45:48.494Z 26 520-539 148 148 148 280 280 280 F F F
161382024-02-13T21:45:48.494Z 27 540-559 146 146 146 275 275 275 F F F
161392024-02-13T21:45:48.494Z 28 560-579 146 146 146 275 275 275 F F F
161402024-02-13T21:45:48.494Z 29 580-599 149 149 149 281 281 281 F F F
161412024-02-13T21:45:48.494ZMax gen: 149, Max flush: 281
161422024-02-13T21:45:48.494Z
161432024-02-13T21:45:48.494ZBegin loop 75
161442024-02-13T21:45:48.494ZDownstairs to restart: 2
161452024-02-13T21:45:48.536Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21497352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
161462024-02-13T21:45:48.537Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215214003Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
161472024-02-13T21:45:48.537Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215234345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
161482024-02-13T21:45:48.546Z{"msg":"Crucible 23f7408a-ca7c-43a7-b5e9-8528e15ed64a has session id: 4019def7-6108-48b4-8223-ddae655ea2ce","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21527712Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738Crucible runtime is spawned
161492024-02-13T21:45:48.546Z,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161502024-02-13T21:45:48.546Z{"msg":"Upstairs opts: Upstairs UUID: 23f7408a-ca7c-43a7-b5e9-8528e15ed64a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215341906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161512024-02-13T21:45:48.546Z{"msg":"Crucible stats registered with UUID: 23f7408a-ca7c-43a7-b5e9-8528e15ed64a","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215385881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161522024-02-13T21:45:48.546Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215427505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
161532024-02-13T21:45:48.546Z{"msg":"23f7408a-ca7c-43a7-b5e9-8528e15ed64a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215555999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161542024-02-13T21:45:48.546Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215614997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161552024-02-13T21:45:48.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215642197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161562024-02-13T21:45:48.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215685212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161572024-02-13T21:45:48.547Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215714611Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161582024-02-13T21:45:48.547Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215740802Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161592024-02-13T21:45:48.548Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215782657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161602024-02-13T21:45:48.548Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21582673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161612024-02-13T21:45:48.548Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215861868Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161622024-02-13T21:45:48.548Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215901303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161632024-02-13T21:45:48.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215928673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161642024-02-13T21:45:48.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215953615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161652024-02-13T21:45:48.548Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.215979485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"io task","client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161662024-02-13T21:45:48.549Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21668273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161672024-02-13T21:45:48.549Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.216722876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161682024-02-13T21:45:48.549Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21673811Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161692024-02-13T21:45:48.549Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.216758792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161702024-02-13T21:45:48.549Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217324117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161712024-02-13T21:45:48.549Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217373909Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161722024-02-13T21:45:48.550Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217797346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161732024-02-13T21:45:48.550Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21783964Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161742024-02-13T21:45:48.550Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217854665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161752024-02-13T21:45:48.550Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21786837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161762024-02-13T21:45:48.550Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217882685Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161772024-02-13T21:45:48.550Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217899638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161782024-02-13T21:45:48.550Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217920381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161792024-02-13T21:45:48.551Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.217933966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161802024-02-13T21:45:48.551Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218192372Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161812024-02-13T21:45:48.551Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218591998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161822024-02-13T21:45:48.551Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218627885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161832024-02-13T21:45:48.551Z{"msg":"[0]R flush_numbers[0..12]: [279, 280, 280, 280, 265, 264, 273, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218644259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161842024-02-13T21:45:48.551Z{"msg":"[0]R generation[0..12]: [147, 148, 148, 148, 140, 140, 145, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218665711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161852024-02-13T21:45:48.552Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218680306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161862024-02-13T21:45:48.552Z{"msg":"[1]R flush_numbers[0..12]: [279, 280, 280, 280, 265, 264, 273, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218701408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161872024-02-13T21:45:48.552Z{"msg":"[1]R generation[0..12]: [147, 148, 148, 148, 140, 140, 145, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218715313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161882024-02-13T21:45:48.552Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218736105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161892024-02-13T21:45:48.552Z{"msg":"[2]R flush_numbers[0..12]: [279, 280, 280, 280, 265, 264, 273, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.21875119Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161902024-02-13T21:45:48.552Z{"msg":"[2]R generation[0..12]: [147, 148, 148, 148, 140, 140, 145, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218771492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161912024-02-13T21:45:48.552Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
161922024-02-13T21:45:48.552Z,"time":"2024-02-13T21:45:47.218785677Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161932024-02-13T21:45:48.553Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218807919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161942024-02-13T21:45:48.553Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218822024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161952024-02-13T21:45:48.553Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218841487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161962024-02-13T21:45:48.553Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218854792Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"":"downstairs","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161972024-02-13T21:45:48.553Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218879903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161982024-02-13T21:45:48.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218892589Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"0","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
161992024-02-13T21:45:48.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218905754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"1","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162002024-02-13T21:45:48.554Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218918829Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"client":"2","":"downstairs","session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162012024-02-13T21:45:48.554Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218933104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162022024-02-13T21:45:48.555Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218952717Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162032024-02-13T21:45:48.555Z{"msg":"23f7408a-ca7c-43a7-b5e9-8528e15ed64a is now active with session: 4019def7-6108-48b4-8223-ddae655ea2ce","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218966322Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162042024-02-13T21:45:48.555Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.218978297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738,"session_id":"4019def7-6108-48b4-8223-ddae655ea2ce"}
162052024-02-13T21:45:48.555Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:47.219008096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
162062024-02-13T21:45:48.555ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162072024-02-13T21:45:48.555ZLoading write count information from file "/tmp/test_repair_verify.data"
162082024-02-13T21:45:48.555ZRun Repair workload
162092024-02-13T21:45:48.555Z01/30 Write block 514 len 24576 data: 17 16 16 19 19 21
162102024-02-13T21:45:48.555Z02/30 Flush
162112024-02-13T21:45:48.555Z03/30 Write block 274 len 28672 data: 15 16 15 16 14 14 12
162122024-02-13T21:45:48.556Z04/30 Read block 484 len 4096
162132024-02-13T21:45:48.556Z05/30 Write block 284 len 36864 data: 8 9 8 8 10 11 10 11 13
162142024-02-13T21:45:48.556Z06/30 Read block 358 len 32768
162152024-02-13T21:45:48.556Z07/30 Read block 420 len 20480
162162024-02-13T21:45:48.556Z08/30 Flush
162172024-02-13T21:45:48.556Z09/30 Write block 135 len 4096 data: 14
162182024-02-13T21:45:48.556Z10/30 Read block 428 len 24576
162192024-02-13T21:45:48.556Z11/30 Write block 402 len 16384 data: 13 11 11 11
162202024-02-13T21:45:48.556Z12/30 Write block 34 len 40960 data: 13 15 14 9 10 11 12 12 10 9
162212024-02-13T21:45:48.556Z13/30 Read block 285 len 40960
162222024-02-13T21:45:48.556Z14/30 Write block 308 len 32768 data: 9 7 8 12 14 13 11 13
162232024-02-13T21:45:48.556Z15/30 Write block 297 len 20480 data: 11 11 10 9 9
162242024-02-13T21:45:48.556Z16/30 Read block 204 len 32768
162252024-02-13T21:45:48.556Z17/30 Read block 526 len 32768
162262024-02-13T21:45:48.556Z18/30 Read block 489 len 24576
162272024-02-13T21:45:48.557Z19/30 Write block 383 len 32768 data: 13 12 12 13 12 11 12 10
162282024-02-13T21:45:48.557Z20/30 Read block 407 len 36864
162292024-02-13T21:45:48.557Z21/30 Read block 115 len 32768
162302024-02-13T21:45:48.557Z22/30 Write block 495 len 12288 data: 10 11 12
162312024-02-13T21:45:48.557Z23/30 Write block 403 len 28672 data: 12 12 12 10 10 12 15
162322024-02-13T21:45:48.557Z24/30 Read block 82 len 4096
162332024-02-13T21:45:48.557Z25/30 Read block 233 len 12288
162342024-02-13T21:45:48.557Z26/30 Write block 571 len 28672 data: 6 6 4 5 5 6 7
162352024-02-13T21:45:48.557Z27/30 Write block 14 len 8192 data: 15 15
162362024-02-13T21:45:48.557Z28/30 Read block 558 len 28672
162372024-02-13T21:45:48.562Z29/30 Write block 248 len 36864 data: 7 10 11 13 10 9 9 11 13
162382024-02-13T21:45:48.563Z30/30 Write block 550 len 28672 data: 12 17 14 13 13 12 13
162392024-02-13T21:45:48.563Z----------------------------------------------------------------
162402024-02-13T21:45:48.563Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:22
162412024-02-13T21:45:48.563ZStates: Active Active Active
162422024-02-13T21:45:48.563ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162432024-02-13T21:45:48.563Z 9 Acked 1008 Write 1 Done Done Sent false
162442024-02-13T21:45:48.563Z 10 Acked 1009 Read 6 Done Done Sent false
162452024-02-13T21:45:48.563Z 11 Acked 1010 Write 4 Done Done Sent false
162462024-02-13T21:45:48.564Z 12 Acked 1011 Write 10 Done Done Sent false
162472024-02-13T21:45:48.564Z 13 Acked 1012 Read 10 Done Done Sent false
162482024-02-13T21:45:48.564Z 14 Acked 1013 Write 8 Done Done Sent false
162492024-02-13T21:45:48.564Z 15 Acked 1014 Write 5 Done Done Sent false
162502024-02-13T21:45:48.564Z 16 Acked 1015 Read 8 Done Done Sent false
162512024-02-13T21:45:48.564Z 17 Acked 1016 Read 8 Done Done Sent false
162522024-02-13T21:45:48.564Z 18 Acked 1017 Read 6 Done Done Sent false
162532024-02-13T21:45:48.564Z 19 Acked 1018 Write 8 Done Done Sent false
162542024-02-13T21:45:48.564Z 20 Acked 1019 Read 9 Done Done Sent false
162552024-02-13T21:45:48.565Z 21 Acked 1020 Read 8 Done Done Sent false
162562024-02-13T21:45:48.565Z 22 Acked 1021 Write 3 Done Done Sent false
162572024-02-13T21:45:48.565Z 23 Acked 1022 Write 7 Done Done Sent false
162582024-02-13T21:45:48.565Z 24 Acked 1023 Read 1 Done Done Sent false
162592024-02-13T21:45:48.565Z 25 Acked 1024 Read 3 Done Done Sent false
162602024-02-13T21:45:48.566Z 26 Acked 1025 Write 7 Done Done Sent false
162612024-02-13T21:45:48.566Z 27 Acked 1026 Write 2 Done Done Sent false
162622024-02-13T21:45:48.566Z 28 Acked 1027 Read 7 Done Done Sent false
162632024-02-13T21:45:48.566Z 29 Acked 1028 Write 9 Sent Sent Sent false
162642024-02-13T21:45:48.566Z 30 Acked 1029 Write 7 Sent Sent Sent false
162652024-02-13T21:45:48.566Z STATES DS:0 DS:1 DS:2 TOTAL
162662024-02-13T21:45:48.566Z New 0 0 0 0
162672024-02-13T21:45:48.566Z Sent 2 2 22 26
162682024-02-13T21:45:48.567Z Done 20 20 0 40
162692024-02-13T21:45:48.567Z Skipped 0 0 0 0
162702024-02-13T21:45:48.567Z Error 0 0 0 0
162712024-02-13T21:45:48.567ZLast Flush: 1007 1007 1007
162722024-02-13T21:45:48.568ZDownstairs last five completed: 1007 1006 1005 1004 1003
162732024-02-13T21:45:48.568ZUpstairs last five completed: 30 29 28 27 26
162742024-02-13T21:45:48.568Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:47.251430666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1738}
162752024-02-13T21:45:48.568ZWrote out file "/tmp/test_repair_verify.data"
162762024-02-13T21:45:48.568Z
162772024-02-13T21:45:48.568ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162782024-02-13T21:45:48.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162792024-02-13T21:45:48.573Z 0 000-019 147 147 147 279 279 279 T T F <---
162802024-02-13T21:45:48.573Z 1 020-039 148 148 148 280 280 280 T T F <---
162812024-02-13T21:45:48.574Z 2 040-059 148 148 148 280 280 280 T T F <---
162822024-02-13T21:45:48.574Z 3 060-079 148 148 148 280 280 280 F F F
162832024-02-13T21:45:48.574Z 4 080-099 140 140 140 265 265 265 F F F
162842024-02-13T21:45:48.574Z 5 100-119 140 140 140 264 264 264 F F F
162852024-02-13T21:45:48.574Z 6 120-139 145 145 145 273 273 273 T T F <---
162862024-02-13T21:45:48.574Z 7 140-159 148 148 148 280 280 280 F F F
162872024-02-13T21:45:48.575Z 8 160-179 146 146 146 276 276 276 F F F
162882024-02-13T21:45:48.575Z 9 180-199 146 146 146 278 278 278 F F F
162892024-02-13T21:45:48.575Z 10 200-219 148 148 148 280 280 280 F F F
162902024-02-13T21:45:48.575Z 11 220-239 149 149 149 281 281 281 F F F
162912024-02-13T21:45:48.576Z 12 240-259 149 149 149 281 281 281 T T F <---
162922024-02-13T21:45:48.576Z 13 260-279 150 150 150 283 283 283 F F F
162932024-02-13T21:45:48.576Z 14 280-299 150 150 150 283 283 283 T T F <---
162942024-02-13T21:45:48.576Z 15 300-319 148 148 148 280 280 280 T T F <---
162952024-02-13T21:45:48.576Z 16 320-339 148 148 148 280 280 280 F F F
162962024-02-13T21:45:48.576Z 17 340-359 149 149 149 281 281 281 F F F
162972024-02-13T21:45:48.576Z 18 360-379 138 138 138 262 262 262 F F F
162982024-02-13T21:45:48.577Z 19 380-399 146 146 146 274 274 274 T T F <---
162992024-02-13T21:45:48.577Z 20 400-419 146 146 146 275 275 275 T T F <---
163002024-02-13T21:45:48.577Z 21 420-439 146 146 146 274 274 274 F F F
163012024-02-13T21:45:48.577Z 22 440-459 148 148 148 280 280 280 F F F
163022024-02-13T21:45:48.577Z 23 460-479 149 149 149 281 281 281 F F F
163032024-02-13T21:45:48.577Z 24 480-499 148 148 148 280 280 280 T T F <---
163042024-02-13T21:45:48.578Z 25 500-519 150 150 150 282 282 282 F F F
163052024-02-13T21:45:48.578Z 26 520-539 148 148 148 280 280 280 F F F
163062024-02-13T21:45:48.578Z 27 540-559 146 146 146 275 275 275 T T F <---
163072024-02-13T21:45:48.578Z 28 560-579 146 146 146 275 275 275 T T F <---
163082024-02-13T21:45:48.578Z 29 580-599 149 149 149 281 281 281 F F F
163092024-02-13T21:45:48.579ZMax gen: 150, Max flush: 283
163102024-02-13T21:45:48.579ZError: Difference in extent metadata found!
163112024-02-13T21:45:48.584ZA Difference in extent metadata is expected here
163122024-02-13T21:45:48.584ZOn loop 75
163132024-02-13T21:45:48.584Z
163142024-02-13T21:45:48.584ZVerifying data now
163152024-02-13T21:45:53.193ZNo change after verify
163162024-02-13T21:45:53.193ZLoop: 75 Downstairs dump after verify (and repair):
163172024-02-13T21:45:53.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163182024-02-13T21:45:53.208Z 0 000-019 151 151 151 284 284 284 F F F
163192024-02-13T21:45:53.208Z 1 020-039 151 151 151 284 284 284 F F F
163202024-02-13T21:45:53.208Z 2 040-059 151 151 151 284 284 284 F F F
163212024-02-13T21:45:53.208Z 3 060-079 148 148 148 280 280 280 F F F
163222024-02-13T21:45:53.208Z 4 080-099 140 140 140 265 265 265 F F F
163232024-02-13T21:45:53.208Z 5 100-119 140 140 140 264 264 264 F F F
163242024-02-13T21:45:53.208Z 6 120-139 151 151 151 284 284 284 F F F
163252024-02-13T21:45:53.209Z 7 140-159 148 148 148 280 280 280 F F F
163262024-02-13T21:45:53.209Z 8 160-179 146 146 146 276 276 276 F F F
163272024-02-13T21:45:53.209Z 9 180-199 146 146 146 278 278 278 F F F
163282024-02-13T21:45:53.209Z 10 200-219 148 148 148 280 280 280 F F F
163292024-02-13T21:45:53.213Z 11 220-239 149 149 149 281 281 281 F F F
163302024-02-13T21:45:53.213Z 12 240-259 151 151 151 284 284 284 F F F
163312024-02-13T21:45:53.213Z 13 260-279 150 150 150 283 283 283 F F F
163322024-02-13T21:45:53.214Z 14 280-299 151 151 151 284 284 284 F F F
163332024-02-13T21:45:53.214Z 15 300-319 151 151 151 284 284 284 F F F
163342024-02-13T21:45:53.214Z 16 320-339 148 148 148 280 280 280 F F F
163352024-02-13T21:45:53.214Z 17 340-359 149 149 149 281 281 281 F F F
163362024-02-13T21:45:53.214Z 18 360-379 138 138 138 262 262 262 F F F
163372024-02-13T21:45:53.215Z 19 380-399 151 151 151 284 284 284 F F F
163382024-02-13T21:45:53.215Z 20 400-419 151 151 151 284 284 284 F F F
163392024-02-13T21:45:53.215Z 21 420-439 146 146 146 274 274 274 F F F
163402024-02-13T21:45:53.215Z 22 440-459 148 148 148 280 280 280 F F F
163412024-02-13T21:45:53.215Z 23 460-479 149 149 149 281 281 281 F F F
163422024-02-13T21:45:53.215Z 24 480-499 151 151 151 284 284 284 F F F
163432024-02-13T21:45:53.215Z 25 500-519 150 150 150 282 282 282 F F F
163442024-02-13T21:45:53.215Z 26 520-539 148 148 148 280 280 280 F F F
163452024-02-13T21:45:53.215Z 27 540-559 151 151 151 284 284 284 F F F
163462024-02-13T21:45:53.216Z 28 560-579 151 151 151 284 284 284 F F F
163472024-02-13T21:45:53.216Z 29 580-599 149 149 149 281 281 281 F F F
163482024-02-13T21:45:53.216ZMax gen: 151, Max flush: 284
163492024-02-13T21:45:53.216Z
163502024-02-13T21:45:53.216ZBegin loop 76
163512024-02-13T21:45:53.216ZDownstairs to restart: 0
163522024-02-13T21:45:53.229Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934031187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
163532024-02-13T21:45:53.229Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934348882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
163542024-02-13T21:45:53.230Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.93438012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
163552024-02-13T21:45:53.237Z{"msg":"Crucible 0bb85612-133c-472d-9378-9a63fe41f32c has session id: 87c610df-6c74-40e8-8dbe-e9cd80076f74","v":0,"name":"crucible","level":30Crucible runtime is spawned
163562024-02-13T21:45:53.237Z,"time":"2024-02-13T21:45:51.934404801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163572024-02-13T21:45:53.237Z{"msg":"Upstairs opts: Upstairs UUID: 0bb85612-133c-472d-9378-9a63fe41f32c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934452964Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163582024-02-13T21:45:53.237Z{"msg":"Crucible stats registered with UUID: 0bb85612-133c-472d-9378-9a63fe41f32c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934498787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163592024-02-13T21:45:53.237Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934524218Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
163602024-02-13T21:45:53.238Z{"msg":"0bb85612-133c-472d-9378-9a63fe41f32c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934594362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163612024-02-13T21:45:53.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934635747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163622024-02-13T21:45:53.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.9346563Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163632024-02-13T21:45:53.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934675863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163642024-02-13T21:45:53.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.93471269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163652024-02-13T21:45:53.238Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934736361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163662024-02-13T21:45:53.238Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934761472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163672024-02-13T21:45:53.239Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934776396Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163682024-02-13T21:45:53.239Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934851129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163692024-02-13T21:45:53.239Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934913267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163702024-02-13T21:45:53.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934955751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163712024-02-13T21:45:53.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.934997386Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163722024-02-13T21:45:53.239Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.93504147Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"io task","client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163732024-02-13T21:45:53.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.935514899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163742024-02-13T21:45:53.239Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.935541809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163752024-02-13T21:45:53.239Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.935557113Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163762024-02-13T21:45:53.240Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.935590261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163772024-02-13T21:45:53.240Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.93606259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163782024-02-13T21:45:53.240Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936207297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163792024-02-13T21:45:53.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936599015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163802024-02-13T21:45:53.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936647158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163812024-02-13T21:45:53.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936677927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163822024-02-13T21:45:53.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936717872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163832024-02-13T21:45:53.241Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936760137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163842024-02-13T21:45:53.241Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.9368073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163852024-02-13T21:45:53.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936843946Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163862024-02-13T21:45:53.241Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.936876805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163872024-02-13T21:45:53.241Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937255417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163882024-02-13T21:45:53.241Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937640508Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163892024-02-13T21:45:53.241Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937673736Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163902024-02-13T21:45:53.241Z{"msg":"[0]R flush_numbers[0..12]: [284, 284, 284, 280, 265, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937695298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163912024-02-13T21:45:53.242Z{"msg":"[0]R generation[0..12]: [151, 151, 151, 148, 140, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.93771753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163922024-02-13T21:45:53.242Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937732584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163932024-02-13T21:45:53.242Z{"msg":"[1]R flush_numbers[0..12]: [284, 284, 284, 280, 265, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937753996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163942024-02-13T21:45:53.242Z{"msg":"[1]R generation[0..12]: [151, 151, 151, 148, 140, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937768451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163952024-02-13T21:45:53.242Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937789074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163962024-02-13T21:45:53.242Z{"msg":"[2]R flush_numbers[0..12]: [284, 284, 284, 280, 265, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937811446Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163972024-02-13T21:45:53.242Z{"msg":"[2]R generation[0..12]: [151, 151, 151, 148, 140, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937832798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163982024-02-13T21:45:53.242Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937847242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
163992024-02-13T21:45:53.242Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937866875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164002024-02-13T21:45:53.242Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
164012024-02-13T21:45:53.243Z,"time":"2024-02-13T21:45:51.93788093Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164022024-02-13T21:45:53.243Z{"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937904462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164032024-02-13T21:45:53.243Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937925724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"":"downstairs","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164042024-02-13T21:45:53.243Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937946257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164052024-02-13T21:45:53.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937959332Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"0","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164062024-02-13T21:45:53.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.937997108Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"1","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164072024-02-13T21:45:53.243Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938010793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"client":"2","":"downstairs","session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164082024-02-13T21:45:53.244Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938025008Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164092024-02-13T21:45:53.244Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938056077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164102024-02-13T21:45:53.244Z{"msg":"0bb85612-133c-472d-9378-9a63fe41f32c is now active with session: 87c610df-6c74-40e8-8dbe-e9cd80076f74","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938085996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164112024-02-13T21:45:53.244Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938118104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746,"session_id":"87c610df-6c74-40e8-8dbe-e9cd80076f74"}
164122024-02-13T21:45:53.244Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:51.938133049Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
164132024-02-13T21:45:53.244ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164142024-02-13T21:45:53.244ZLoading write count information from file "/tmp/test_repair_verify.data"
164152024-02-13T21:45:53.244ZRun Repair workload
164162024-02-13T21:45:53.244Z01/30 Write block 61 len 28672 data: 15 13 11 8 9 9 10
164172024-02-13T21:45:53.249Z02/30 Read block 185 len 28672
164182024-02-13T21:45:53.249Z03/30 Flush
164192024-02-13T21:45:53.250Z04/30 Write block 84 len 4096 data: 9
164202024-02-13T21:45:53.250Z05/30 Read block 27 len 8192
164212024-02-13T21:45:53.250Z06/30 Read block 318 len 4096
164222024-02-13T21:45:53.250Z07/30 Write block 364 len 24576 data: 12 13 12 15 14 14
164232024-02-13T21:45:53.250Z08/30 Read block 312 len 8192
164242024-02-13T21:45:53.250Z09/30 Write block 48 len 28672 data: 9 10 10 10 10 10 11
164252024-02-13T21:45:53.250Z10/30 Read block 238 len 8192
164262024-02-13T21:45:53.251Z11/30 Read block 410 len 32768
164272024-02-13T21:45:53.251Z12/30 Write block 419 len 28672 data: 17 14 10 10 13 11 11
164282024-02-13T21:45:53.251Z13/30 Read block 559 len 16384
164292024-02-13T21:45:53.251Z14/30 Read block 321 len 24576
164302024-02-13T21:45:53.251Z15/30 Write block 12 len 4096 data: 13
164312024-02-13T21:45:53.251Z16/30 Read block 279 len 40960
164322024-02-13T21:45:53.251Z17/30 Write block 343 len 40960 data: 15 13 12 11 12 12 10 10 10 12
164332024-02-13T21:45:53.251Z18/30 Read block 4 len 32768
164342024-02-13T21:45:53.256Z19/30 Write block 277 len 8192 data: 17 15
164352024-02-13T21:45:53.256Z20/30 Read block 581 len 16384
164362024-02-13T21:45:53.256Z21/30 Read block 173 len 24576
164372024-02-13T21:45:53.256Z22/30 Read block 259 len 36864
164382024-02-13T21:45:53.256Z23/30 Flush
164392024-02-13T21:45:53.273Z24/30 Write block 484 len 4096 data: 10
164402024-02-13T21:45:53.273Z25/30 Write block 18 len 20480 data: 13 11 12 12 14
164412024-02-13T21:45:53.273Z26/30 Read block 365 len 8192
164422024-02-13T21:45:53.273Z27/30 Read block 561 len 36864
164432024-02-13T21:45:53.273Z28/30 Read block 511 len 8192
164442024-02-13T21:45:53.274Z29/30 Read block 56 len 12288
164452024-02-13T21:45:53.278Z30/30 Read block 265 len 24576
164462024-02-13T21:45:53.278Z----------------------------------------------------------------
164472024-02-13T21:45:53.278Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:27
164482024-02-13T21:45:53.278ZStates: Active Active Active
164492024-02-13T21:45:53.278ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164502024-02-13T21:45:53.279Z 4 Acked 1003 Write 1 Sent Done Done false
164512024-02-13T21:45:53.279Z 5 Acked 1004 Read 2 Sent Done Done false
164522024-02-13T21:45:53.279Z 6 Acked 1005 Read 1 Sent Done Done false
164532024-02-13T21:45:53.279Z 7 Acked 1006 Write 6 Sent Done Done false
164542024-02-13T21:45:53.279Z 8 Acked 1007 Read 2 Sent Done Done false
164552024-02-13T21:45:53.279Z 9 Acked 1008 Write 7 Sent Done Done false
164562024-02-13T21:45:53.279Z 10 Acked 1009 Read 2 Sent Done Done false
164572024-02-13T21:45:53.280Z 11 Acked 1010 Read 8 Sent Done Done false
164582024-02-13T21:45:53.280Z 12 Acked 1011 Write 7 Sent Done Done false
164592024-02-13T21:45:53.280Z 13 Acked 1012 Read 4 Sent Done Done false
164602024-02-13T21:45:53.280Z 14 Acked 1013 Read 6 Sent Done Done false
164612024-02-13T21:45:53.280Z 15 Acked 1014 Write 1 Sent Done Done false
164622024-02-13T21:45:53.280Z 16 Acked 1015 Read 10 Sent Done Done false
164632024-02-13T21:45:53.280Z 17 Acked 1016 Write 10 Sent Done Done false
164642024-02-13T21:45:53.280Z 18 Acked 1017 Read 8 Sent Done Done false
164652024-02-13T21:45:53.280Z 19 Acked 1018 Write 2 Sent Done Done false
164662024-02-13T21:45:53.281Z 20 Acked 1019 Read 4 Sent Done Done false
164672024-02-13T21:45:53.281Z 21 Acked 1020 Read 6 Sent Done Done false
164682024-02-13T21:45:53.281Z 22 Acked 1021 Read 9 Sent Done Done false
164692024-02-13T21:45:53.281Z 23 Acked 1022 Flush 0 Sent Done Done false
164702024-02-13T21:45:53.281Z 24 Acked 1023 Write 1 Sent Done Done false
164712024-02-13T21:45:53.281Z 25 Acked 1024 Write 5 Sent Done Done false
164722024-02-13T21:45:53.282Z 26 Acked 1025 Read 2 Sent Done Done false
164732024-02-13T21:45:53.282Z 27 Acked 1026 Read 9 Sent Done Done false
164742024-02-13T21:45:53.282Z 28 Acked 1027 Read 2 Sent Done Done false
164752024-02-13T21:45:53.282Z 29 Acked 1028 Read 3 Sent Done Done false
164762024-02-13T21:45:53.282Z 30 Acked 1029 Read 6 Sent Done Done false
164772024-02-13T21:45:53.282Z STATES DS:0 DS:1 DS:2 TOTAL
164782024-02-13T21:45:53.282Z New 0 0 0 0
164792024-02-13T21:45:53.282Z Sent 27 0 0 27
164802024-02-13T21:45:53.282Z Done 0 27 27 54
164812024-02-13T21:45:53.282Z Skipped 0 0 0 0
164822024-02-13T21:45:53.283Z Error 0 0 0 0
164832024-02-13T21:45:53.283ZLast Flush: 1002 1022 1022
164842024-02-13T21:45:53.283ZDownstairs last five completed: 1002 1001 1000
164852024-02-13T21:45:53.283ZUpstairs last five completed: 30 29 28 27 26
164862024-02-13T21:45:53.283Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:51.967797709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1746}
164872024-02-13T21:45:53.283ZWrote out file "/tmp/test_repair_verify.data"
164882024-02-13T21:45:53.283Z
164892024-02-13T21:45:53.283ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164902024-02-13T21:45:53.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164912024-02-13T21:45:53.288Z 0 000-019 151 152 152 284 286 286 F T T <---
164922024-02-13T21:45:53.289Z 1 020-039 151 151 151 284 284 284 F T T <---
164932024-02-13T21:45:53.289Z 2 040-059 151 152 152 284 286 286 F F F <---
164942024-02-13T21:45:53.289Z 3 060-079 152 152 152 285 285 285 F F F
164952024-02-13T21:45:53.289Z 4 080-099 140 152 152 265 286 286 F F F <---
164962024-02-13T21:45:53.289Z 5 100-119 140 140 140 264 264 264 F F F
164972024-02-13T21:45:53.289Z 6 120-139 151 151 151 284 284 284 F F F
164982024-02-13T21:45:53.289Z 7 140-159 148 148 148 280 280 280 F F F
164992024-02-13T21:45:53.289Z 8 160-179 146 146 146 276 276 276 F F F
165002024-02-13T21:45:53.290Z 9 180-199 146 146 146 278 278 278 F F F
165012024-02-13T21:45:53.290Z 10 200-219 148 148 148 280 280 280 F F F
165022024-02-13T21:45:53.290Z 11 220-239 149 149 149 281 281 281 F F F
165032024-02-13T21:45:53.290Z 12 240-259 151 151 151 284 284 284 F F F
165042024-02-13T21:45:53.290Z 13 260-279 150 152 152 283 286 286 F F F <---
165052024-02-13T21:45:53.290Z 14 280-299 151 151 151 284 284 284 F F F
165062024-02-13T21:45:53.290Z 15 300-319 151 151 151 284 284 284 F F F
165072024-02-13T21:45:53.290Z 16 320-339 148 148 148 280 280 280 F F F
165082024-02-13T21:45:53.290Z 17 340-359 149 152 152 281 286 286 F F F <---
165092024-02-13T21:45:53.290Z 18 360-379 138 152 152 262 286 286 F F F <---
165102024-02-13T21:45:53.290Z 19 380-399 151 151 151 284 284 284 F F F
165112024-02-13T21:45:53.291Z 20 400-419 151 152 152 284 286 286 F F F <---
165122024-02-13T21:45:53.291Z 21 420-439 146 152 152 274 286 286 F F F <---
165132024-02-13T21:45:53.291Z 22 440-459 148 148 148 280 280 280 F F F
165142024-02-13T21:45:53.291Z 23 460-479 149 149 149 281 281 281 F F F
165152024-02-13T21:45:53.291Z 24 480-499 151 151 151 284 284 284 F T T <---
165162024-02-13T21:45:53.291Z 25 500-519 150 150 150 282 282 282 F F F
165172024-02-13T21:45:53.291Z 26 520-539 148 148 148 280 280 280 F F F
165182024-02-13T21:45:53.292Z 27 540-559 151 151 151 284 284 284 F F F
165192024-02-13T21:45:53.292Z 28 560-579 151 151 151 284 284 284 F F F
165202024-02-13T21:45:53.292Z 29 580-599 149 149 149 281 281 281 F F F
165212024-02-13T21:45:53.292ZMax gen: 152, Max flush: 286
165222024-02-13T21:45:53.292ZError: Difference in extent metadata found!
165232024-02-13T21:45:53.292ZA Difference in extent metadata is expected here
165242024-02-13T21:45:53.292ZOn loop 76
165252024-02-13T21:45:53.292Z
165262024-02-13T21:45:53.297ZVerifying data now
165272024-02-13T21:45:57.801ZNo change after verify
165282024-02-13T21:45:57.801ZLoop: 76 Downstairs dump after verify (and repair):
165292024-02-13T21:45:57.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165302024-02-13T21:45:57.816Z 0 000-019 153 153 153 287 287 287 F F F
165312024-02-13T21:45:57.816Z 1 020-039 153 153 153 287 287 287 F F F
165322024-02-13T21:45:57.816Z 2 040-059 152 152 152 286 286 286 F F F
165332024-02-13T21:45:57.816Z 3 060-079 152 152 152 285 285 285 F F F
165342024-02-13T21:45:57.816Z 4 080-099 152 152 152 286 286 286 F F F
165352024-02-13T21:45:57.816Z 5 100-119 140 140 140 264 264 264 F F F
165362024-02-13T21:45:57.817Z 6 120-139 151 151 151 284 284 284 F F F
165372024-02-13T21:45:57.817Z 7 140-159 148 148 148 280 280 280 F F F
165382024-02-13T21:45:57.817Z 8 160-179 146 146 146 276 276 276 F F F
165392024-02-13T21:45:57.817Z 9 180-199 146 146 146 278 278 278 F F F
165402024-02-13T21:45:57.817Z 10 200-219 148 148 148 280 280 280 F F F
165412024-02-13T21:45:57.817Z 11 220-239 149 149 149 281 281 281 F F F
165422024-02-13T21:45:57.817Z 12 240-259 151 151 151 284 284 284 F F F
165432024-02-13T21:45:57.821Z 13 260-279 152 152 152 286 286 286 F F F
165442024-02-13T21:45:57.821Z 14 280-299 151 151 151 284 284 284 F F F
165452024-02-13T21:45:57.822Z 15 300-319 151 151 151 284 284 284 F F F
165462024-02-13T21:45:57.822Z 16 320-339 148 148 148 280 280 280 F F F
165472024-02-13T21:45:57.822Z 17 340-359 152 152 152 286 286 286 F F F
165482024-02-13T21:45:57.822Z 18 360-379 152 152 152 286 286 286 F F F
165492024-02-13T21:45:57.822Z 19 380-399 151 151 151 284 284 284 F F F
165502024-02-13T21:45:57.822Z 20 400-419 152 152 152 286 286 286 F F F
165512024-02-13T21:45:57.822Z 21 420-439 152 152 152 286 286 286 F F F
165522024-02-13T21:45:57.822Z 22 440-459 148 148 148 280 280 280 F F F
165532024-02-13T21:45:57.822Z 23 460-479 149 149 149 281 281 281 F F F
165542024-02-13T21:45:57.823Z 24 480-499 153 153 153 287 287 287 F F F
165552024-02-13T21:45:57.823Z 25 500-519 150 150 150 282 282 282 F F F
165562024-02-13T21:45:57.823Z 26 520-539 148 148 148 280 280 280 F F F
165572024-02-13T21:45:57.823Z 27 540-559 151 151 151 284 284 284 F F F
165582024-02-13T21:45:57.823Z 28 560-579 151 151 151 284 284 284 F F F
165592024-02-13T21:45:57.823Z 29 580-599 149 149 149 281 281 281 F F F
165602024-02-13T21:45:57.823ZMax gen: 153, Max flush: 287
165612024-02-13T21:45:57.823Z
165622024-02-13T21:45:57.823ZBegin loop 77
165632024-02-13T21:45:57.823ZDownstairs to restart: 2
165642024-02-13T21:45:57.836Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540432869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
165652024-02-13T21:45:57.836Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540687657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
165662024-02-13T21:45:57.844Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540715387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
165672024-02-13T21:45:57.844ZCrucible runtime is spawned
165682024-02-13T21:45:57.844Z{"msg":"Crucible 2cabb1be-9364-4988-b56e-801dfeabf651 has session id: 599eaaa4-b996-40bd-91ac-2bd2d31d3f98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540796457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165692024-02-13T21:45:57.844Z{"msg":"Upstairs opts: Upstairs UUID: 2cabb1be-9364-4988-b56e-801dfeabf651, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54084328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165702024-02-13T21:45:57.844Z{"msg":"Crucible stats registered with UUID: 2cabb1be-9364-4988-b56e-801dfeabf651","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540877478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165712024-02-13T21:45:57.845Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540911516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
165722024-02-13T21:45:57.845Z{"msg":"2cabb1be-9364-4988-b56e-801dfeabf651 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540942444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165732024-02-13T21:45:57.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.540975153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165742024-02-13T21:45:57.845Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541001863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165752024-02-13T21:45:57.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541028613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165762024-02-13T21:45:57.845Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541067889Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165772024-02-13T21:45:57.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54109299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165782024-02-13T21:45:57.846Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541137124Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165792024-02-13T21:45:57.846Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541163374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165802024-02-13T21:45:57.846Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541205439Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165812024-02-13T21:45:57.846Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54123099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165822024-02-13T21:45:57.846Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541265777Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165832024-02-13T21:45:57.846Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541290438Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165842024-02-13T21:45:57.846Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541332873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"io task","client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165852024-02-13T21:45:57.846Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541838669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165862024-02-13T21:45:57.846Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541873776Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165872024-02-13T21:45:57.847Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541888781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165882024-02-13T21:45:57.847Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.541911803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165892024-02-13T21:45:57.847Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.542475348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165902024-02-13T21:45:57.847Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.542518773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165912024-02-13T21:45:57.847Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.542930723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165922024-02-13T21:45:57.847Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543062545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165932024-02-13T21:45:57.847Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543099512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165942024-02-13T21:45:57.847Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543116066Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165952024-02-13T21:45:57.848Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543130291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165962024-02-13T21:45:57.848Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543150284Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165972024-02-13T21:45:57.848Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543164308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165982024-02-13T21:45:57.848Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543189599Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
165992024-02-13T21:45:57.848Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54346362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166002024-02-13T21:45:57.848Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543856787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166012024-02-13T21:45:57.848Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543891984Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166022024-02-13T21:45:57.848Z{"msg":"[0]R flush_numbers[0..12]: [287, 287, 286, 285, 286, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543909078Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166032024-02-13T21:45:57.849Z{"msg":"[0]R generation[0..12]: [153, 153, 152, 152, 152, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543930241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166042024-02-13T21:45:57.849Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543945265Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166052024-02-13T21:45:57.849Z{"msg":"[1]R flush_numbers[0..12]: [287, 287, 286, 285, 286, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543966837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166062024-02-13T21:45:57.849Z{"msg":"[1]R generation[0..12]: [153, 153, 152, 152, 152, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.543982542Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166072024-02-13T21:45:57.849Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544003274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166082024-02-13T21:45:57.849Z{"msg":"[2]R flush_numbers[0..12]: [287, 287, 286, 285, 286, 264, 284, 280, 276, 278, 280, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544017749Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166092024-02-13T21:45:57.849Z{"msg":"Wait for a query_work_queue command to finish before sending IO
166102024-02-13T21:45:57.849Z[2]R generation[0..12]: [153, 153, 152, 152, 152, 140, 151, 148, 146, 146, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544039801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166112024-02-13T21:45:57.850Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544054316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166122024-02-13T21:45:57.850Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54406814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166132024-02-13T21:45:57.850Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54409758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166142024-02-13T21:45:57.850Z{"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544111665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166152024-02-13T21:45:57.850Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.54412567Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"":"downstairs","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166162024-02-13T21:45:57.850Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544146162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166172024-02-13T21:45:57.850Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544158678Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"0","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166182024-02-13T21:45:57.850Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544171963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"1","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166192024-02-13T21:45:57.850Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544185578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"client":"2","":"downstairs","session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166202024-02-13T21:45:57.851Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544198793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166212024-02-13T21:45:57.851Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544218136Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166222024-02-13T21:45:57.851Z{"msg":"2cabb1be-9364-4988-b56e-801dfeabf651 is now active with session: 599eaaa4-b996-40bd-91ac-2bd2d31d3f98","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544231721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166232024-02-13T21:45:57.851Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544245416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754,"session_id":"599eaaa4-b996-40bd-91ac-2bd2d31d3f98"}
166242024-02-13T21:45:57.851Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:45:56.544264699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
166252024-02-13T21:45:57.851ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166262024-02-13T21:45:57.861ZLoading write count information from file "/tmp/test_repair_verify.data"
166272024-02-13T21:45:57.861ZRun Repair workload
166282024-02-13T21:45:57.861Z01/30 Write block 222 len 8192 data: 17 18
166292024-02-13T21:45:57.861Z02/30 Write block 411 len 8192 data: 15 10
166302024-02-13T21:45:57.861Z03/30 Flush
166312024-02-13T21:45:57.861Z04/30 Write block 408 len 4096 data: 13
166322024-02-13T21:45:57.861Z05/30 Read block 99 len 32768
166332024-02-13T21:45:57.861Z06/30 Read block 105 len 20480
166342024-02-13T21:45:57.861Z07/30 Write block 464 len 24576 data: 14 15 13 14 11 12
166352024-02-13T21:45:57.861Z08/30 Read block 365 len 4096
166362024-02-13T21:45:57.861Z09/30 Read block 525 len 8192
166372024-02-13T21:45:57.861Z10/30 Flush
166382024-02-13T21:45:57.862Z11/30 Write block 580 len 36864 data: 5 6 7 9 10 11 11 11 10
166392024-02-13T21:45:57.862Z12/30 Write block 270 len 36864 data: 12 12 13 14 16 17 16 18 16
166402024-02-13T21:45:57.862Z13/30 Read block 195 len 16384
166412024-02-13T21:45:57.862Z14/30 Read block 49 len 12288
166422024-02-13T21:45:57.862Z15/30 Flush
166432024-02-13T21:45:57.862Z16/30 Write block 69 len 24576 data: 15 14 12 14 12 13
166442024-02-13T21:45:57.862Z17/30 Write block 393 len 28672 data: 10 11 9 9 8 9 9
166452024-02-13T21:45:57.862Z18/30 Read block 179 len 8192
166462024-02-13T21:45:57.895Z19/30 Read block 550 len 8192
166472024-02-13T21:45:57.895Z20/30 Flush
166482024-02-13T21:45:57.895Z21/30 Write block 525 len 12288 data: 8 8 10
166492024-02-13T21:45:57.895Z22/30 Write block 427 len 40960 data: 11 13 17 15 15 16 13 14 13 8
166502024-02-13T21:45:57.895Z23/30 Read block 116 len 20480
166512024-02-13T21:45:57.895Z24/30 Read block 488 len 20480
166522024-02-13T21:45:57.896Z25/30 Read block 242 len 8192
166532024-02-13T21:45:57.896Z26/30 Write block 37 len 8192 data: 10 11
166542024-02-13T21:45:57.896Z27/30 Read block 418 len 12288
166552024-02-13T21:45:57.896Z28/30 Write block 279 len 36864 data: 15 13 9 7 7 9 10 9 9
166562024-02-13T21:45:57.901Z29/30 Read block 456 len 8192
166572024-02-13T21:45:57.901Z30/30 Read block 368 len 28672
166582024-02-13T21:45:57.901Z----------------------------------------------------------------
166592024-02-13T21:45:57.901Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166602024-02-13T21:45:57.902ZStates: Active Active Active
166612024-02-13T21:45:57.902ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166622024-02-13T21:45:57.902Z 1 Acked 1000 Write 2 Done Done Sent false
166632024-02-13T21:45:57.902Z 2 Acked 1001 Write 2 Done Done Sent false
166642024-02-13T21:45:57.902Z 3 Acked 1002 Flush 0 Done Done Sent false
166652024-02-13T21:45:57.902Z 4 Acked 1003 Write 1 Done Done Sent false
166662024-02-13T21:45:57.902Z 5 Acked 1004 Read 8 Done Done Sent false
166672024-02-13T21:45:57.902Z 6 Acked 1005 Read 5 Done Done Sent false
166682024-02-13T21:45:57.902Z 7 Acked 1006 Write 6 Done Done Sent false
166692024-02-13T21:45:57.902Z 8 Acked 1007 Read 1 Done Done Sent false
166702024-02-13T21:45:57.902Z 9 Acked 1008 Read 2 Done Done Sent false
166712024-02-13T21:45:57.902Z 10 Acked 1009 Flush 0 Done Done Sent false
166722024-02-13T21:45:57.903Z 11 Acked 1010 Write 9 Done Done Sent false
166732024-02-13T21:45:57.903Z 12 Acked 1011 Write 9 Done Done Sent false
166742024-02-13T21:45:57.903Z 13 Acked 1012 Read 4 Done Done Sent false
166752024-02-13T21:45:57.903Z 14 Acked 1013 Read 3 Done Done Sent false
166762024-02-13T21:45:57.903Z 15 Acked 1014 Flush 0 Done Done Sent false
166772024-02-13T21:45:57.903Z 16 Acked 1015 Write 6 Done Done Sent false
166782024-02-13T21:45:57.903Z 17 Acked 1016 Write 7 Done Done Sent false
166792024-02-13T21:45:57.903Z 18 Acked 1017 Read 2 Done Done Sent false
166802024-02-13T21:45:57.903Z 19 Acked 1018 Read 2 Done Done Sent false
166812024-02-13T21:45:57.903Z 20 Acked 1019 Flush 0 Done Done Sent false
166822024-02-13T21:45:57.904Z 21 Acked 1020 Write 3 Done Done Sent false
166832024-02-13T21:45:57.904Z 22 Acked 1021 Write 10 Done Done Sent false
166842024-02-13T21:45:57.904Z 23 Acked 1022 Read 5 Done Done Sent false
166852024-02-13T21:45:57.904Z 24 Acked 1023 Read 5 Done Done Sent false
166862024-02-13T21:45:57.904Z 25 Acked 1024 Read 2 Done Done Sent false
166872024-02-13T21:45:57.904Z 26 Acked 1025 Write 2 Done Done Sent false
166882024-02-13T21:45:57.904Z 27 Acked 1026 Read 3 Done Done Sent false
166892024-02-13T21:45:57.904Z 28 Acked 1027 Write 9 Done Done Sent false
166902024-02-13T21:45:57.904Z 29 Acked 1028 Read 2 Done Done Sent false
166912024-02-13T21:45:57.904Z 30 Acked 1029 Read 7 Done Sent Sent false
166922024-02-13T21:45:57.904Z STATES DS:0 DS:1 DS:2 TOTAL
166932024-02-13T21:45:57.905Z New 0 0 0 0
166942024-02-13T21:45:57.905Z Sent 0 1 30 31
166952024-02-13T21:45:57.905Z Done 30 29 0 59
166962024-02-13T21:45:57.905Z Skipped 0 0 0 0
166972024-02-13T21:45:57.905Z Error 0 0 0 0
166982024-02-13T21:45:57.905ZLast Flush: 1019 1019 0
166992024-02-13T21:45:57.905ZDownstairs last five completed:
167002024-02-13T21:45:57.905ZUpstairs last five completed: 30 29 28 27 26
167012024-02-13T21:45:57.905Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:45:56.575153715Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1754}
167022024-02-13T21:45:57.905ZWrote out file "/tmp/test_repair_verify.data"
167032024-02-13T21:45:57.905Z
167042024-02-13T21:45:57.906ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167052024-02-13T21:45:57.906ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167062024-02-13T21:45:57.906Z 0 000-019 153 153 153 287 287 287 F F F
167072024-02-13T21:45:57.906Z 1 020-039 153 153 153 287 287 287 T T F <---
167082024-02-13T21:45:57.906Z 2 040-059 152 152 152 286 286 286 F F F
167092024-02-13T21:45:57.906Z 3 060-079 154 154 152 291 291 285 F F F <---
167102024-02-13T21:45:57.906Z 4 080-099 152 152 152 286 286 286 F F F
167112024-02-13T21:45:57.906Z 5 100-119 140 140 140 264 264 264 F F F
167122024-02-13T21:45:57.906Z 6 120-139 151 151 151 284 284 284 F F F
167132024-02-13T21:45:57.906Z 7 140-159 148 148 148 280 280 280 F F F
167142024-02-13T21:45:57.906Z 8 160-179 146 146 146 276 276 276 F F F
167152024-02-13T21:45:57.906Z 9 180-199 146 146 146 278 278 278 F F F
167162024-02-13T21:45:57.907Z 10 200-219 148 148 148 280 280 280 F F F
167172024-02-13T21:45:57.907Z 11 220-239 154 154 149 288 288 281 F F F <---
167182024-02-13T21:45:57.907Z 12 240-259 151 151 151 284 284 284 F F F
167192024-02-13T21:45:57.907Z 13 260-279 154 154 152 290 290 286 T T F <---
167202024-02-13T21:45:57.907Z 14 280-299 151 151 151 284 284 284 T T F <---
167212024-02-13T21:45:57.907Z 15 300-319 151 151 151 284 284 284 F F F
167222024-02-13T21:45:57.907Z 16 320-339 148 148 148 280 280 280 F F F
167232024-02-13T21:45:57.907Z 17 340-359 152 152 152 286 286 286 F F F
167242024-02-13T21:45:57.907Z 18 360-379 152 152 152 286 286 286 F F F
167252024-02-13T21:45:57.907Z 19 380-399 154 154 151 291 291 284 F F F <---
167262024-02-13T21:45:57.908Z 20 400-419 154 154 152 289 289 286 F F F <---
167272024-02-13T21:45:57.908Z 21 420-439 152 152 152 286 286 286 T T F <---
167282024-02-13T21:45:57.908Z 22 440-459 148 148 148 280 280 280 F F F
167292024-02-13T21:45:57.908Z 23 460-479 154 154 149 289 289 281 F F F <---
167302024-02-13T21:45:57.908Z 24 480-499 153 153 153 287 287 287 F F F
167312024-02-13T21:45:57.908Z 25 500-519 150 150 150 282 282 282 F F F
167322024-02-13T21:45:57.908Z 26 520-539 148 148 148 280 280 280 T T F <---
167332024-02-13T21:45:57.908Z 27 540-559 151 151 151 284 284 284 F F F
167342024-02-13T21:45:57.908Z 28 560-579 151 151 151 284 284 284 F F F
167352024-02-13T21:45:57.908Z 29 580-599 154 154 149 290 290 281 F F F <---
167362024-02-13T21:45:57.908ZMax gen: 154, Max flush: 291
167372024-02-13T21:45:57.908ZError: Difference in extent metadata found!
167382024-02-13T21:45:57.909ZA Difference in extent metadata is expected here
167392024-02-13T21:45:57.909ZOn loop 77
167402024-02-13T21:45:57.909Z
167412024-02-13T21:45:57.909ZVerifying data now
167422024-02-13T21:46:02.460ZNo change after verify
167432024-02-13T21:46:02.460ZLoop: 77 Downstairs dump after verify (and repair):
167442024-02-13T21:46:02.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167452024-02-13T21:46:02.475Z 0 000-019 153 153 153 287 287 287 F F F
167462024-02-13T21:46:02.475Z 1 020-039 155 155 155 292 292 292 F F F
167472024-02-13T21:46:02.475Z 2 040-059 152 152 152 286 286 286 F F F
167482024-02-13T21:46:02.476Z 3 060-079 154 154 154 291 291 291 F F F
167492024-02-13T21:46:02.476Z 4 080-099 152 152 152 286 286 286 F F F
167502024-02-13T21:46:02.476Z 5 100-119 140 140 140 264 264 264 F F F
167512024-02-13T21:46:02.476Z 6 120-139 151 151 151 284 284 284 F F F
167522024-02-13T21:46:02.476Z 7 140-159 148 148 148 280 280 280 F F F
167532024-02-13T21:46:02.476Z 8 160-179 146 146 146 276 276 276 F F F
167542024-02-13T21:46:02.476Z 9 180-199 146 146 146 278 278 278 F F F
167552024-02-13T21:46:02.476Z 10 200-219 148 148 148 280 280 280 F F F
167562024-02-13T21:46:02.476Z 11 220-239 154 154 154 288 288 288 F F F
167572024-02-13T21:46:02.477Z 12 240-259 151 151 151 284 284 284 F F F
167582024-02-13T21:46:02.477Z 13 260-279 155 155 155 292 292 292 F F F
167592024-02-13T21:46:02.477Z 14 280-299 155 155 155 292 292 292 F F F
167602024-02-13T21:46:02.477Z 15 300-319 151 151 151 284 284 284 F F F
167612024-02-13T21:46:02.477Z 16 320-339 148 148 148 280 280 280 F F F
167622024-02-13T21:46:02.484Z 17 340-359 152 152 152 286 286 286 F F F
167632024-02-13T21:46:02.484Z 18 360-379 152 152 152 286 286 286 F F F
167642024-02-13T21:46:02.484Z 19 380-399 154 154 154 291 291 291 F F F
167652024-02-13T21:46:02.484Z 20 400-419 154 154 154 289 289 289 F F F
167662024-02-13T21:46:02.484Z 21 420-439 155 155 155 292 292 292 F F F
167672024-02-13T21:46:02.484Z 22 440-459 148 148 148 280 280 280 F F F
167682024-02-13T21:46:02.485Z 23 460-479 154 154 154 289 289 289 F F F
167692024-02-13T21:46:02.485Z 24 480-499 153 153 153 287 287 287 F F F
167702024-02-13T21:46:02.485Z 25 500-519 150 150 150 282 282 282 F F F
167712024-02-13T21:46:02.485Z 26 520-539 155 155 155 292 292 292 F F F
167722024-02-13T21:46:02.485Z 27 540-559 151 151 151 284 284 284 F F F
167732024-02-13T21:46:02.485Z 28 560-579 151 151 151 284 284 284 F F F
167742024-02-13T21:46:02.485Z 29 580-599 154 154 154 290 290 290 F F F
167752024-02-13T21:46:02.485ZMax gen: 155, Max flush: 292
167762024-02-13T21:46:02.485Z
167772024-02-13T21:46:02.486ZBegin loop 78
167782024-02-13T21:46:02.486ZDownstairs to restart: 2
167792024-02-13T21:46:02.496Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.196911316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
167802024-02-13T21:46:02.496Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197173131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
167812024-02-13T21:46:02.496Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197207388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
167822024-02-13T21:46:02.505Z{"msg":"Crucible e2cdeefd-34a5-4dc9-959b-585054c8921b has session id: f15662a6-7672-4780-ab13-c482d2f89850","v":0,"name":"crucible","level":30Crucible runtime is spawned
167832024-02-13T21:46:02.505Z,"time":"2024-02-13T21:46:01.197231509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167842024-02-13T21:46:02.505Z{"msg":"Upstairs opts: Upstairs UUID: e2cdeefd-34a5-4dc9-959b-585054c8921b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.19728532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167852024-02-13T21:46:02.505Z{"msg":"Crucible stats registered with UUID: e2cdeefd-34a5-4dc9-959b-585054c8921b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.19731346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167862024-02-13T21:46:02.505Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.19734144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
167872024-02-13T21:46:02.505Z{"msg":"e2cdeefd-34a5-4dc9-959b-585054c8921b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197429688Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167882024-02-13T21:46:02.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197473692Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167892024-02-13T21:46:02.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197502351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167902024-02-13T21:46:02.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197537878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167912024-02-13T21:46:02.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197566058Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167922024-02-13T21:46:02.506Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197605494Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167932024-02-13T21:46:02.506Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197632824Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167942024-02-13T21:46:02.506Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197666612Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167952024-02-13T21:46:02.506Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.19769941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167962024-02-13T21:46:02.506Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197737116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167972024-02-13T21:46:02.507Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197811539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167982024-02-13T21:46:02.507Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197859122Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
167992024-02-13T21:46:02.507Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.197902436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"io task","client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168002024-02-13T21:46:02.507Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.198493312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168012024-02-13T21:46:02.507Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.198531128Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168022024-02-13T21:46:02.507Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.198550061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168032024-02-13T21:46:02.507Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.198592786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168042024-02-13T21:46:02.507Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199026069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168052024-02-13T21:46:02.507Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199141267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168062024-02-13T21:46:02.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199511143Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168072024-02-13T21:46:02.508Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199554087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168082024-02-13T21:46:02.508Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199583547Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168092024-02-13T21:46:02.508Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199642005Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168102024-02-13T21:46:02.508Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199680012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168112024-02-13T21:46:02.508Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199696266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168122024-02-13T21:46:02.508Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199782394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168132024-02-13T21:46:02.508Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.199809704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168142024-02-13T21:46:02.508Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200267878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168152024-02-13T21:46:02.509Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.20064798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168162024-02-13T21:46:02.509Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200679659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168172024-02-13T21:46:02.509Z{"msg":"[0]R flush_numbers[0..12]: [287, 292, 286, 291, 286, 264, 284, 280, 276, 278, 280, 288]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200694524Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168182024-02-13T21:46:02.509Z{"msg":"[0]R generation[0..12]: [153, 155, 152, 154, 152, 140, 151, 148, 146, 146, 148, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200716755Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168192024-02-13T21:46:02.509Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.20073253Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168202024-02-13T21:46:02.509Z{"msg":"[1]R flush_numbers[0..12]: [287, 292, 286, 291, 286, 264, 284, 280, 276, 278, 280, 288]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200754162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168212024-02-13T21:46:02.509Z{"msg":"[1]R generation[0..12]: [153, 155, 152, 154, 152, 140, 151, 148, 146, 146, 148, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200768937Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168222024-02-13T21:46:02.509Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200789459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168232024-02-13T21:46:02.510Z{"msg":"[2]R flush_numbers[0..12]: [287, 292, 286, 291, 286, 264, 284, 280, 276, 278, 280, 288]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200803804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168242024-02-13T21:46:02.510Z{"msg":"[2]R generation[0..12]: [153, 155, 152, 154, 152, 140, 151, 148, 146, 146, 148, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200829924Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168252024-02-13T21:46:02.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
168262024-02-13T21:46:02.510Z,"time":"2024-02-13T21:46:01.200844359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168272024-02-13T21:46:02.510Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200867061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168282024-02-13T21:46:02.510Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200880606Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168292024-02-13T21:46:02.510Z{"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200905127Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168302024-02-13T21:46:02.510Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200918632Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"":"downstairs","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168312024-02-13T21:46:02.510Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200932077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168322024-02-13T21:46:02.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200944883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"0","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168332024-02-13T21:46:02.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200958358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"1","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168342024-02-13T21:46:02.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200971993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"client":"2","":"downstairs","session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168352024-02-13T21:46:02.511Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.200993125Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168362024-02-13T21:46:02.511Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.20100613Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168372024-02-13T21:46:02.511Z{"msg":"e2cdeefd-34a5-4dc9-959b-585054c8921b is now active with session: f15662a6-7672-4780-ab13-c482d2f89850","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.201019426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168382024-02-13T21:46:02.511Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.201038059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762,"session_id":"f15662a6-7672-4780-ab13-c482d2f89850"}
168392024-02-13T21:46:02.511Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:01.201050634Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
168402024-02-13T21:46:02.511ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168412024-02-13T21:46:02.512ZLoading write count information from file "/tmp/test_repair_verify.data"
168422024-02-13T21:46:02.512ZRun Repair workload
168432024-02-13T21:46:02.512Z01/30 Write block 537 len 24576 data: 13 16 19 21 18 16
168442024-02-13T21:46:02.519Z02/30 Read block 46 len 24576
168452024-02-13T21:46:02.519Z03/30 Read block 564 len 24576
168462024-02-13T21:46:02.520Z04/30 Read block 310 len 16384
168472024-02-13T21:46:02.520Z05/30 Read block 131 len 28672
168482024-02-13T21:46:02.520Z06/30 Flush
168492024-02-13T21:46:02.520Z07/30 Write block 447 len 28672 data: 15 13 16 16 14 11 12
168502024-02-13T21:46:02.520Z08/30 Flush
168512024-02-13T21:46:02.521Z09/30 Write block 527 len 40960 data: 11 10 9 8 11 12 12 13 13 11
168522024-02-13T21:46:02.521Z10/30 Read block 403 len 28672
168532024-02-13T21:46:02.521Z11/30 Read block 377 len 16384
168542024-02-13T21:46:02.521Z12/30 Read block 401 len 32768
168552024-02-13T21:46:02.521Z13/30 Write block 246 len 28672 data: 8 8 8 11 12 14 11
168562024-02-13T21:46:02.522Z14/30 Write block 147 len 40960 data: 11 10 10 9 9 8 11 11 12 10
168572024-02-13T21:46:02.522Z15/30 Write block 66 len 36864 data: 10 11 12 16 15 13 15 13 14
168582024-02-13T21:46:02.522Z16/30 Read block 90 len 40960
168592024-02-13T21:46:02.522Z17/30 Flush
168602024-02-13T21:46:02.527Z18/30 Write block 567 len 8192 data: 12 12
168612024-02-13T21:46:02.527Z19/30 Write block 235 len 40960 data: 10 8 8 12 12 13 13 14 13 10
168622024-02-13T21:46:02.527Z20/30 Write block 461 len 28672 data: 16 15 13 15 16 14 15
168632024-02-13T21:46:02.527Z21/30 Write block 269 len 28672 data: 12 13 13 14 15 17 18
168642024-02-13T21:46:02.527Z22/30 Read block 403 len 28672
168652024-02-13T21:46:02.527Z23/30 Write block 182 len 28672 data: 8 11 12 14 11 10 13
168662024-02-13T21:46:02.528Z24/30 Read block 352 len 16384
168672024-02-13T21:46:02.533Z25/30 Write block 528 len 36864 data: 11 10 9 12 13 13 14 14 12
168682024-02-13T21:46:02.533Z26/30 Write block 25 len 8192 data: 12 10
168692024-02-13T21:46:02.533Z27/30 Write block 316 len 8192 data: 14 14
168702024-02-13T21:46:02.533Z28/30 Read block 267 len 24576
168712024-02-13T21:46:02.533Z29/30 Read block 55 len 16384
168722024-02-13T21:46:02.533Z30/30 Read block 184 len 12288
168732024-02-13T21:46:02.534Z----------------------------------------------------------------
168742024-02-13T21:46:02.534Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:22
168752024-02-13T21:46:02.534ZStates: Active Active Active
168762024-02-13T21:46:02.534ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168772024-02-13T21:46:02.534Z 9 Acked 1008 Write 10 Done Done Sent false
168782024-02-13T21:46:02.534Z 10 Acked 1009 Read 7 Done Done Sent false
168792024-02-13T21:46:02.534Z 11 Acked 1010 Read 4 Done Done Sent false
168802024-02-13T21:46:02.534Z 12 Acked 1011 Read 8 Done Done Sent false
168812024-02-13T21:46:02.535Z 13 Acked 1012 Write 7 Done Done Sent false
168822024-02-13T21:46:02.535Z 14 Acked 1013 Write 10 Done Done Sent false
168832024-02-13T21:46:02.535Z 15 Acked 1014 Write 9 Done Done Sent false
168842024-02-13T21:46:02.535Z 16 Acked 1015 Read 10 Done Done Sent false
168852024-02-13T21:46:02.535Z 17 Acked 1016 Flush 0 Done Done Sent false
168862024-02-13T21:46:02.535Z 18 Acked 1017 Write 2 Done Done Sent false
168872024-02-13T21:46:02.536Z 19 Acked 1018 Write 10 Done Done Sent false
168882024-02-13T21:46:02.536Z 20 Acked 1019 Write 7 Done Done Sent false
168892024-02-13T21:46:02.536Z 21 Acked 1020 Write 7 Done Done Sent false
168902024-02-13T21:46:02.536Z 22 Acked 1021 Read 7 Done Done Sent false
168912024-02-13T21:46:02.536Z 23 Acked 1022 Write 7 Done Done Sent false
168922024-02-13T21:46:02.536Z 24 Acked 1023 Read 4 Done Done Sent false
168932024-02-13T21:46:02.536Z 25 Acked 1024 Write 9 Done Done Sent false
168942024-02-13T21:46:02.537Z 26 Acked 1025 Write 2 Done Done Sent false
168952024-02-13T21:46:02.537Z 27 Acked 1026 Write 2 Done Done Sent false
168962024-02-13T21:46:02.537Z 28 Acked 1027 Read 6 Done Done Sent false
168972024-02-13T21:46:02.537Z 29 Acked 1028 Read 4 Done Done Sent false
168982024-02-13T21:46:02.537Z 30 Acked 1029 Read 3 Done Done Sent false
168992024-02-13T21:46:02.537Z STATES DS:0 DS:1 DS:2 TOTAL
169002024-02-13T21:46:02.537Z New 0 0 0 0
169012024-02-13T21:46:02.538Z Sent 0 0 22 22
169022024-02-13T21:46:02.538Z Done 22 22 0 44
169032024-02-13T21:46:02.538Z Skipped 0 0 0 0
169042024-02-13T21:46:02.538Z Error 0 0 0 0
169052024-02-13T21:46:02.538ZLast Flush: 1016 1016 1007
169062024-02-13T21:46:02.538ZDownstairs last five completed: 1007 1006 1005 1004 1003
169072024-02-13T21:46:02.538ZUpstairs last five completed: 30 29 28 27 26
169082024-02-13T21:46:02.538Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:01.233915603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1762}
169092024-02-13T21:46:02.539ZWrote out file "/tmp/test_repair_verify.data"
169102024-02-13T21:46:02.544Z
169112024-02-13T21:46:02.544ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169122024-02-13T21:46:02.558ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169132024-02-13T21:46:02.558Z 0 000-019 153 153 153 287 287 287 F F F
169142024-02-13T21:46:02.558Z 1 020-039 155 155 155 292 292 292 T T F <---
169152024-02-13T21:46:02.558Z 2 040-059 152 152 152 286 286 286 F F F
169162024-02-13T21:46:02.558Z 3 060-079 156 156 154 295 295 291 F F F <---
169172024-02-13T21:46:02.558Z 4 080-099 152 152 152 286 286 286 F F F
169182024-02-13T21:46:02.559Z 5 100-119 140 140 140 264 264 264 F F F
169192024-02-13T21:46:02.559Z 6 120-139 151 151 151 284 284 284 F F F
169202024-02-13T21:46:02.559Z 7 140-159 156 156 148 295 295 280 F F F <---
169212024-02-13T21:46:02.559Z 8 160-179 146 146 146 276 276 276 F F F
169222024-02-13T21:46:02.565Z 9 180-199 146 146 146 278 278 278 T T F <---
169232024-02-13T21:46:02.565Z 10 200-219 148 148 148 280 280 280 F F F
169242024-02-13T21:46:02.565Z 11 220-239 154 154 154 288 288 288 T T F <---
169252024-02-13T21:46:02.565Z 12 240-259 156 156 151 295 295 284 T T F <---
169262024-02-13T21:46:02.565Z 13 260-279 155 155 155 292 292 292 T T F <---
169272024-02-13T21:46:02.565Z 14 280-299 155 155 155 292 292 292 F F F
169282024-02-13T21:46:02.565Z 15 300-319 151 151 151 284 284 284 T T F <---
169292024-02-13T21:46:02.565Z 16 320-339 148 148 148 280 280 280 F F F
169302024-02-13T21:46:02.566Z 17 340-359 152 152 152 286 286 286 F F F
169312024-02-13T21:46:02.566Z 18 360-379 152 152 152 286 286 286 F F F
169322024-02-13T21:46:02.566Z 19 380-399 154 154 154 291 291 291 F F F
169332024-02-13T21:46:02.566Z 20 400-419 154 154 154 289 289 289 F F F
169342024-02-13T21:46:02.566Z 21 420-439 155 155 155 292 292 292 F F F
169352024-02-13T21:46:02.566Z 22 440-459 156 156 156 294 294 294 F F F
169362024-02-13T21:46:02.566Z 23 460-479 154 154 154 289 289 289 T T F <---
169372024-02-13T21:46:02.567Z 24 480-499 153 153 153 287 287 287 F F F
169382024-02-13T21:46:02.567Z 25 500-519 150 150 150 282 282 282 F F F
169392024-02-13T21:46:02.567Z 26 520-539 156 156 156 295 295 293 T T F <---
169402024-02-13T21:46:02.567Z 27 540-559 156 156 156 293 293 293 F F F
169412024-02-13T21:46:02.567Z 28 560-579 151 151 151 284 284 284 T T F <---
169422024-02-13T21:46:02.567Z 29 580-599 154 154 154 290 290 290 F F F
169432024-02-13T21:46:02.567ZMax gen: 156, Max flush: 295
169442024-02-13T21:46:02.567ZError: Difference in extent metadata found!
169452024-02-13T21:46:02.567ZA Difference in extent metadata is expected here
169462024-02-13T21:46:02.568ZOn loop 78
169472024-02-13T21:46:02.568Z
169482024-02-13T21:46:02.568ZVerifying data now
169492024-02-13T21:46:07.114ZNo change after verify
169502024-02-13T21:46:07.114ZLoop: 78 Downstairs dump after verify (and repair):
169512024-02-13T21:46:07.130ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169522024-02-13T21:46:07.130Z 0 000-019 153 153 153 287 287 287 F F F
169532024-02-13T21:46:07.130Z 1 020-039 157 157 157 296 296 296 F F F
169542024-02-13T21:46:07.130Z 2 040-059 152 152 152 286 286 286 F F F
169552024-02-13T21:46:07.130Z 3 060-079 156 156 156 295 295 295 F F F
169562024-02-13T21:46:07.135Z 4 080-099 152 152 152 286 286 286 F F F
169572024-02-13T21:46:07.135Z 5 100-119 140 140 140 264 264 264 F F F
169582024-02-13T21:46:07.135Z 6 120-139 151 151 151 284 284 284 F F F
169592024-02-13T21:46:07.135Z 7 140-159 156 156 156 295 295 295 F F F
169602024-02-13T21:46:07.136Z 8 160-179 146 146 146 276 276 276 F F F
169612024-02-13T21:46:07.136Z 9 180-199 157 157 157 296 296 296 F F F
169622024-02-13T21:46:07.136Z 10 200-219 148 148 148 280 280 280 F F F
169632024-02-13T21:46:07.136Z 11 220-239 157 157 157 296 296 296 F F F
169642024-02-13T21:46:07.136Z 12 240-259 157 157 157 296 296 296 F F F
169652024-02-13T21:46:07.136Z 13 260-279 157 157 157 296 296 296 F F F
169662024-02-13T21:46:07.137Z 14 280-299 155 155 155 292 292 292 F F F
169672024-02-13T21:46:07.137Z 15 300-319 157 157 157 296 296 296 F F F
169682024-02-13T21:46:07.137Z 16 320-339 148 148 148 280 280 280 F F F
169692024-02-13T21:46:07.137Z 17 340-359 152 152 152 286 286 286 F F F
169702024-02-13T21:46:07.137Z 18 360-379 152 152 152 286 286 286 F F F
169712024-02-13T21:46:07.137Z 19 380-399 154 154 154 291 291 291 F F F
169722024-02-13T21:46:07.137Z 20 400-419 154 154 154 289 289 289 F F F
169732024-02-13T21:46:07.137Z 21 420-439 155 155 155 292 292 292 F F F
169742024-02-13T21:46:07.138Z 22 440-459 156 156 156 294 294 294 F F F
169752024-02-13T21:46:07.138Z 23 460-479 157 157 157 296 296 296 F F F
169762024-02-13T21:46:07.138Z 24 480-499 153 153 153 287 287 287 F F F
169772024-02-13T21:46:07.138Z 25 500-519 150 150 150 282 282 282 F F F
169782024-02-13T21:46:07.138Z 26 520-539 157 157 157 296 296 296 F F F
169792024-02-13T21:46:07.138Z 27 540-559 156 156 156 293 293 293 F F F
169802024-02-13T21:46:07.138Z 28 560-579 157 157 157 296 296 296 F F F
169812024-02-13T21:46:07.139Z 29 580-599 154 154 154 290 290 290 F F F
169822024-02-13T21:46:07.139ZMax gen: 157, Max flush: 296
169832024-02-13T21:46:07.139Z
169842024-02-13T21:46:07.139ZBegin loop 79
169852024-02-13T21:46:07.139ZDownstairs to restart: 1
169862024-02-13T21:46:07.150Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.850624266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
169872024-02-13T21:46:07.151Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
169882024-02-13T21:46:07.151Z,"time":"2024-02-13T21:46:05.850952867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
169892024-02-13T21:46:07.158Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.850987555Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
169902024-02-13T21:46:07.159Z{"msg":"Crucible 54a32e47-4686-4ab4-8ca0-ad9850898a91 has session id: 895f2030-cf98-4389-b754-7f6ab7a7ac93","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851021562Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169912024-02-13T21:46:07.159Z{"msg":"Upstairs opts: Upstairs UUID: 54a32e47-4686-4ab4-8ca0-ad9850898a91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851056819Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169922024-02-13T21:46:07.159Z{"msg":"Crucible stats registered with UUID: 54a32e47-4686-4ab4-8ca0-ad9850898a91","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851094246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169932024-02-13T21:46:07.159Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851112369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
169942024-02-13T21:46:07.159Z{"msg":"54a32e47-4686-4ab4-8ca0-ad9850898a91 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851148096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169952024-02-13T21:46:07.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851170428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169962024-02-13T21:46:07.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85122146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169972024-02-13T21:46:07.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851258086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169982024-02-13T21:46:07.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851293114Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
169992024-02-13T21:46:07.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851328741Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170002024-02-13T21:46:07.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851354302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170012024-02-13T21:46:07.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851377753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170022024-02-13T21:46:07.161Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851408362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170032024-02-13T21:46:07.161Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851434213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170042024-02-13T21:46:07.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851473418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170052024-02-13T21:46:07.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85149663Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170062024-02-13T21:46:07.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.851573992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"io task","client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170072024-02-13T21:46:07.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.852121733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170082024-02-13T21:46:07.162Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.852170735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170092024-02-13T21:46:07.162Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.852217018Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170102024-02-13T21:46:07.162Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.852246618Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170112024-02-13T21:46:07.162Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.852695835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170122024-02-13T21:46:07.162Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85284735Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170132024-02-13T21:46:07.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85320473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170142024-02-13T21:46:07.163Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853245116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170152024-02-13T21:46:07.163Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853271056Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170162024-02-13T21:46:07.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853294747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170172024-02-13T21:46:07.163Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853327066Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170182024-02-13T21:46:07.163Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853351697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170192024-02-13T21:46:07.163Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85345404Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170202024-02-13T21:46:07.163Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853513958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170212024-02-13T21:46:07.163Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.853946681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170222024-02-13T21:46:07.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854361891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170232024-02-13T21:46:07.164Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854408074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170242024-02-13T21:46:07.164Z{"msg":"[0]R flush_numbers[0..12]: [287, 296, 286, 295, 286, 264, 284, 295, 276, 296, 280, 296]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85444465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170252024-02-13T21:46:07.164Z{"msg":"[0]R generation[0..12]: [153, 157, 152, 156, 152, 140, 151, 156, 146, 157, 148, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854481117Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170262024-02-13T21:46:07.164Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854516084Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170272024-02-13T21:46:07.164Z{"msg":"[1]R flush_numbers[0..12]: [287, 296, 286, 295, 286, 264, 284, 295, 276, 296, 280, 296]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854552861Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170282024-02-13T21:46:07.164Z{"msg":"Wait for a query_work_queue command to finish before sending IO
170292024-02-13T21:46:07.164Z[1]R generation[0..12]: [153, 157, 152, 156, 152, 140, 151, 156, 146, 157, 148, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854592657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170302024-02-13T21:46:07.164Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854624225Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170312024-02-13T21:46:07.165Z{"msg":"[2]R flush_numbers[0..12]: [287, 296, 286, 295, 286, 264, 284, 295, 276, 296, 280, 296]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854671838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170322024-02-13T21:46:07.165Z{"msg":"[2]R generation[0..12]: [153, 157, 152, 156, 152, 140, 151, 156, 146, 157, 148, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854719371Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170332024-02-13T21:46:07.165Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854762315Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170342024-02-13T21:46:07.165Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854798232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170352024-02-13T21:46:07.165Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854828511Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170362024-02-13T21:46:07.165Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854842836Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170372024-02-13T21:46:07.166Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854862939Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"":"downstairs","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170382024-02-13T21:46:07.166Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854876854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170392024-02-13T21:46:07.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854889619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"0","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170402024-02-13T21:46:07.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854909592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"1","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170412024-02-13T21:46:07.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854923387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"client":"2","":"downstairs","session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170422024-02-13T21:46:07.166Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.85494321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170432024-02-13T21:46:07.166Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854966231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170442024-02-13T21:46:07.166Z{"msg":"54a32e47-4686-4ab4-8ca0-ad9850898a91 is now active with session: 895f2030-cf98-4389-b754-7f6ab7a7ac93","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.854982775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170452024-02-13T21:46:07.167Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.855007766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770,"session_id":"895f2030-cf98-4389-b754-7f6ab7a7ac93"}
170462024-02-13T21:46:07.167Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:05.855032247Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
170472024-02-13T21:46:07.167ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170482024-02-13T21:46:07.167ZLoading write count information from file "/tmp/test_repair_verify.data"
170492024-02-13T21:46:07.167ZRun Repair workload
170502024-02-13T21:46:07.172Z01/30 Write block 32 len 24576 data: 12 12 14 16 15 11
170512024-02-13T21:46:07.172Z02/30 Read block 343 len 24576
170522024-02-13T21:46:07.173Z03/30 Read block 562 len 32768
170532024-02-13T21:46:07.173Z04/30 Read block 374 len 24576
170542024-02-13T21:46:07.173Z05/30 Read block 422 len 8192
170552024-02-13T21:46:07.173Z06/30 Read block 249 len 4096
170562024-02-13T21:46:07.173Z07/30 Read block 413 len 4096
170572024-02-13T21:46:07.173Z08/30 Write block 499 len 40960 data: 9 10 6 8 7 9 8 12 12 12
170582024-02-13T21:46:07.173Z09/30 Read block 511 len 16384
170592024-02-13T21:46:07.173Z10/30 Read block 187 len 4096
170602024-02-13T21:46:07.174Z11/30 Write block 316 len 4096 data: 15
170612024-02-13T21:46:07.174Z12/30 Read block 83 len 40960
170622024-02-13T21:46:07.174Z13/30 Read block 145 len 8192
170632024-02-13T21:46:07.174Z14/30 Write block 466 len 8192 data: 15 16
170642024-02-13T21:46:07.174Z15/30 Read block 566 len 36864
170652024-02-13T21:46:07.174Z16/30 Read block 231 len 16384
170662024-02-13T21:46:07.174Z17/30 Write block 61 len 16384 data: 16 14 12 9
170672024-02-13T21:46:07.174Z18/30 Write block 466 len 28672 data: 16 17 12 13 13 15 17
170682024-02-13T21:46:07.174Z19/30 Read block 255 len 24576
170692024-02-13T21:46:07.175Z20/30 Write block 23 len 24576 data: 12 12 13 11 8 8
170702024-02-13T21:46:07.175Z21/30 Read block 331 len 24576
170712024-02-13T21:46:07.179Z22/30 Read block 302 len 16384
170722024-02-13T21:46:07.180Z23/30 Write block 272 len 32768 data: 15 16 18 19 17 19 17 16
170732024-02-13T21:46:07.180Z24/30 Write block 160 len 36864 data: 9 11 11 10 8 6 8 5 4
170742024-02-13T21:46:07.180Z25/30 Write block 249 len 20480 data: 12 13 15 12 10
170752024-02-13T21:46:07.180Z26/30 Flush
170762024-02-13T21:46:07.184Z27/30 Write block 220 len 40960 data: 18 16 18 19 17 15 12 13 12 10
170772024-02-13T21:46:07.184Z28/30 Write block 273 len 36864 data: 17 19 20 18 20 18 17 14 10
170782024-02-13T21:46:07.184Z29/30 Read block 350 len 24576
170792024-02-13T21:46:07.184Z30/30 Read block 573 len 20480
170802024-02-13T21:46:07.309Z----------------------------------------------------------------
170812024-02-13T21:46:07.309Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170822024-02-13T21:46:07.309ZStates: Active Active Active
170832024-02-13T21:46:07.310ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170842024-02-13T21:46:07.310Z 1 Acked 1000 Write 6 Done Sent Done false
170852024-02-13T21:46:07.310Z 2 Acked 1001 Read 6 Done Sent Done false
170862024-02-13T21:46:07.310Z 3 Acked 1002 Read 8 Done Sent Done false
170872024-02-13T21:46:07.310Z 4 Acked 1003 Read 6 Done Sent Done false
170882024-02-13T21:46:07.310Z 5 Acked 1004 Read 2 Done Sent Done false
170892024-02-13T21:46:07.310Z 6 Acked 1005 Read 1 Done Sent Done false
170902024-02-13T21:46:07.311Z 7 Acked 1006 Read 1 Done Sent Done false
170912024-02-13T21:46:07.311Z 8 Acked 1007 Write 10 Done Sent Done false
170922024-02-13T21:46:07.311Z 9 Acked 1008 Read 4 Done Sent Done false
170932024-02-13T21:46:07.311Z 10 Acked 1009 Read 1 Done Sent Done false
170942024-02-13T21:46:07.311Z 11 Acked 1010 Write 1 Done Sent Done false
170952024-02-13T21:46:07.311Z 12 Acked 1011 Read 10 Done Sent Done false
170962024-02-13T21:46:07.311Z 13 Acked 1012 Read 2 Done Sent Done false
170972024-02-13T21:46:07.311Z 14 Acked 1013 Write 2 Done Sent Done false
170982024-02-13T21:46:07.312Z 15 Acked 1014 Read 9 Done Sent Done false
170992024-02-13T21:46:07.312Z 16 Acked 1015 Read 4 Done Sent Done false
171002024-02-13T21:46:07.312Z 17 Acked 1016 Write 4 Done Sent Done false
171012024-02-13T21:46:07.312Z 18 Acked 1017 Write 7 Done Sent Done false
171022024-02-13T21:46:07.312Z 19 Acked 1018 Read 6 Done Sent Done false
171032024-02-13T21:46:07.312Z 20 Acked 1019 Write 6 Done Sent Done false
171042024-02-13T21:46:07.313Z 21 Acked 1020 Read 6 Done Sent Done false
171052024-02-13T21:46:07.313Z 22 Acked 1021 Read 4 Done Sent Done false
171062024-02-13T21:46:07.313Z 23 Acked 1022 Write 8 Done Sent Done false
171072024-02-13T21:46:07.313Z 24 Acked 1023 Write 9 Done Sent Done false
171082024-02-13T21:46:07.313Z 25 Acked 1024 Write 5 Done Sent Done false
171092024-02-13T21:46:07.313Z 26 Acked 1025 Flush 0 Done Sent Done false
171102024-02-13T21:46:07.313Z 27 Acked 1026 Write 10 Done Sent Done false
171112024-02-13T21:46:07.314Z 28 Acked 1027 Write 9 Done Sent Done false
171122024-02-13T21:46:07.314Z 29 Acked 1028 Read 6 Done Sent Done false
171132024-02-13T21:46:07.314Z 30 Acked 1029 Read 5 Done Sent Done false
171142024-02-13T21:46:07.314Z STATES DS:0 DS:1 DS:2 TOTAL
171152024-02-13T21:46:07.314Z New 0 0 0 0
171162024-02-13T21:46:07.314Z Sent 0 30 0 30
171172024-02-13T21:46:07.314Z Done 30 0 30 60
171182024-02-13T21:46:07.315Z Skipped 0 0 0 0
171192024-02-13T21:46:07.315Z Error 0 0 0 0
171202024-02-13T21:46:07.315ZLast Flush: 1025 0 1025
171212024-02-13T21:46:07.315ZDownstairs last five completed:
171222024-02-13T21:46:07.317ZUpstairs last five completed: 30 29 28 27 26
171232024-02-13T21:46:07.317Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:05.885341733Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1770}
171242024-02-13T21:46:07.317ZWrote out file "/tmp/test_repair_verify.data"
171252024-02-13T21:46:07.322Z
171262024-02-13T21:46:07.323ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171272024-02-13T21:46:07.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171282024-02-13T21:46:07.323Z 0 000-019 153 153 153 287 287 287 F F F
171292024-02-13T21:46:07.323Z 1 020-039 158 157 158 297 296 297 F F F <---
171302024-02-13T21:46:07.323Z 2 040-059 152 152 152 286 286 286 F F F
171312024-02-13T21:46:07.323Z 3 060-079 158 156 158 297 295 297 F F F <---
171322024-02-13T21:46:07.323Z 4 080-099 152 152 152 286 286 286 F F F
171332024-02-13T21:46:07.323Z 5 100-119 140 140 140 264 264 264 F F F
171342024-02-13T21:46:07.323Z 6 120-139 151 151 151 284 284 284 F F F
171352024-02-13T21:46:07.323Z 7 140-159 156 156 156 295 295 295 F F F
171362024-02-13T21:46:07.324Z 8 160-179 158 146 158 297 276 297 F F F <---
171372024-02-13T21:46:07.324Z 9 180-199 157 157 157 296 296 296 F F F
171382024-02-13T21:46:07.324Z 10 200-219 148 148 148 280 280 280 F F F
171392024-02-13T21:46:07.324Z 11 220-239 157 157 157 296 296 296 T F T <---
171402024-02-13T21:46:07.324Z 12 240-259 158 157 158 297 296 297 F F F <---
171412024-02-13T21:46:07.324Z 13 260-279 158 157 158 297 296 297 T F T <---
171422024-02-13T21:46:07.324Z 14 280-299 155 155 155 292 292 292 T F T <---
171432024-02-13T21:46:07.324Z 15 300-319 158 157 158 297 296 297 F F F <---
171442024-02-13T21:46:07.325Z 16 320-339 148 148 148 280 280 280 F F F
171452024-02-13T21:46:07.325Z 17 340-359 152 152 152 286 286 286 F F F
171462024-02-13T21:46:07.325Z 18 360-379 152 152 152 286 286 286 F F F
171472024-02-13T21:46:07.325Z 19 380-399 154 154 154 291 291 291 F F F
171482024-02-13T21:46:07.325Z 20 400-419 154 154 154 289 289 289 F F F
171492024-02-13T21:46:07.325Z 21 420-439 155 155 155 292 292 292 F F F
171502024-02-13T21:46:07.326Z 22 440-459 156 156 156 294 294 294 F F F
171512024-02-13T21:46:07.326Z 23 460-479 158 157 158 297 296 297 F F F <---
171522024-02-13T21:46:07.326Z 24 480-499 158 153 158 297 287 297 F F F <---
171532024-02-13T21:46:07.326Z 25 500-519 158 150 158 297 282 297 F F F <---
171542024-02-13T21:46:07.326Z 26 520-539 157 157 157 296 296 296 F F F
171552024-02-13T21:46:07.326Z 27 540-559 156 156 156 293 293 293 F F F
171562024-02-13T21:46:07.326Z 28 560-579 157 157 157 296 296 296 F F F
171572024-02-13T21:46:07.327Z 29 580-599 154 154 154 290 290 290 F F F
171582024-02-13T21:46:07.327ZMax gen: 158, Max flush: 297
171592024-02-13T21:46:07.327ZError: Difference in extent metadata found!
171602024-02-13T21:46:07.327ZA Difference in extent metadata is expected here
171612024-02-13T21:46:07.327ZOn loop 79
171622024-02-13T21:46:07.327Z
171632024-02-13T21:46:07.327ZVerifying data now
171642024-02-13T21:46:11.769ZNo change after verify
171652024-02-13T21:46:11.769ZLoop: 79 Downstairs dump after verify (and repair):
171662024-02-13T21:46:11.783ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171672024-02-13T21:46:11.783Z 0 000-019 153 153 153 287 287 287 F F F
171682024-02-13T21:46:11.783Z 1 020-039 158 158 158 297 297 297 F F F
171692024-02-13T21:46:11.783Z 2 040-059 152 152 152 286 286 286 F F F
171702024-02-13T21:46:11.783Z 3 060-079 158 158 158 297 297 297 F F F
171712024-02-13T21:46:11.783Z 4 080-099 152 152 152 286 286 286 F F F
171722024-02-13T21:46:11.784Z 5 100-119 140 140 140 264 264 264 F F F
171732024-02-13T21:46:11.784Z 6 120-139 151 151 151 284 284 284 F F F
171742024-02-13T21:46:11.784Z 7 140-159 156 156 156 295 295 295 F F F
171752024-02-13T21:46:11.784Z 8 160-179 158 158 158 297 297 297 F F F
171762024-02-13T21:46:11.784Z 9 180-199 157 157 157 296 296 296 F F F
171772024-02-13T21:46:11.784Z 10 200-219 148 148 148 280 280 280 F F F
171782024-02-13T21:46:11.787Z 11 220-239 159 159 159 298 298 298 F F F
171792024-02-13T21:46:11.787Z 12 240-259 158 158 158 297 297 297 F F F
171802024-02-13T21:46:11.787Z 13 260-279 159 159 159 298 298 298 F F F
171812024-02-13T21:46:11.787Z 14 280-299 159 159 159 298 298 298 F F F
171822024-02-13T21:46:11.787Z 15 300-319 158 158 158 297 297 297 F F F
171832024-02-13T21:46:11.787Z 16 320-339 148 148 148 280 280 280 F F F
171842024-02-13T21:46:11.788Z 17 340-359 152 152 152 286 286 286 F F F
171852024-02-13T21:46:11.788Z 18 360-379 152 152 152 286 286 286 F F F
171862024-02-13T21:46:11.788Z 19 380-399 154 154 154 291 291 291 F F F
171872024-02-13T21:46:11.788Z 20 400-419 154 154 154 289 289 289 F F F
171882024-02-13T21:46:11.788Z 21 420-439 155 155 155 292 292 292 F F F
171892024-02-13T21:46:11.788Z 22 440-459 156 156 156 294 294 294 F F F
171902024-02-13T21:46:11.788Z 23 460-479 158 158 158 297 297 297 F F F
171912024-02-13T21:46:11.788Z 24 480-499 158 158 158 297 297 297 F F F
171922024-02-13T21:46:11.788Z 25 500-519 158 158 158 297 297 297 F F F
171932024-02-13T21:46:11.788Z 26 520-539 157 157 157 296 296 296 F F F
171942024-02-13T21:46:11.788Z 27 540-559 156 156 156 293 293 293 F F F
171952024-02-13T21:46:11.788Z 28 560-579 157 157 157 296 296 296 F F F
171962024-02-13T21:46:11.788Z 29 580-599 154 154 154 290 290 290 F F F
171972024-02-13T21:46:11.788ZMax gen: 159, Max flush: 298
171982024-02-13T21:46:11.788Z
171992024-02-13T21:46:11.788ZBegin loop 80
172002024-02-13T21:46:11.788ZDownstairs to restart: 2
172012024-02-13T21:46:11.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503250851Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
172022024-02-13T21:46:11.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503509807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
172032024-02-13T21:46:11.809Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503565617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
172042024-02-13T21:46:11.809ZCrucible runtime is spawned
172052024-02-13T21:46:11.809Z{"msg":"Crucible 3e5444dd-66c0-4c4a-af6b-fe1317a23aa2 has session id: f0d0735e-038d-4fd5-826c-cda882d7e5c5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503607872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172062024-02-13T21:46:11.809Z{"msg":"Upstairs opts: Upstairs UUID: 3e5444dd-66c0-4c4a-af6b-fe1317a23aa2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503655025Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172072024-02-13T21:46:11.809Z{"msg":"Crucible stats registered with UUID: 3e5444dd-66c0-4c4a-af6b-fe1317a23aa2","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503684314Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172082024-02-13T21:46:11.809Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503709075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
172092024-02-13T21:46:11.809Z{"msg":"3e5444dd-66c0-4c4a-af6b-fe1317a23aa2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503789296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172102024-02-13T21:46:11.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503816416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172112024-02-13T21:46:11.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50383241Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172122024-02-13T21:46:11.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503853653Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172132024-02-13T21:46:11.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503867917Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172142024-02-13T21:46:11.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503885791Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172152024-02-13T21:46:11.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503912631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172162024-02-13T21:46:11.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.503928695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172172024-02-13T21:46:11.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50399709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172182024-02-13T21:46:11.810Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.504042234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172192024-02-13T21:46:11.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.504229766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172202024-02-13T21:46:11.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.504264414Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172212024-02-13T21:46:11.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.504289934Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"io task","client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172222024-02-13T21:46:11.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50510198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172232024-02-13T21:46:11.810Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505142605Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172242024-02-13T21:46:11.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505174254Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172252024-02-13T21:46:11.810Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505259872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172262024-02-13T21:46:11.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505716907Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172272024-02-13T21:46:11.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505796968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172282024-02-13T21:46:11.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505836803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172292024-02-13T21:46:11.810Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.505854257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172302024-02-13T21:46:11.810Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506357534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172312024-02-13T21:46:11.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50647882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172322024-02-13T21:46:11.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506522565Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172332024-02-13T21:46:11.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506540578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172342024-02-13T21:46:11.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506569947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172352024-02-13T21:46:11.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506612592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172362024-02-13T21:46:11.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506650538Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172372024-02-13T21:46:11.815Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.506999222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172382024-02-13T21:46:11.815Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507037078Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172392024-02-13T21:46:11.815Z{"msg":"[0]R flush_numbers[0..12]: [287, 297, 286, 297, 286, 264, 284, 295, 297, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507054632Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172402024-02-13T21:46:11.815Z{"msg":"[0]R generation[0..12]: [153, 158, 152, 158, 152, 140, 151, 156, 158, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507079352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172412024-02-13T21:46:11.815Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507094647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172422024-02-13T21:46:11.815Z{"msg":"[1]R flush_numbers[0..12]: [287, 297, 286, 297, 286, 264, 284, 295, 297, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507109981Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172432024-02-13T21:46:11.815Z{"msg":"[1]R generation[0..12]: [153, 158, 152, 158, 152, 140, 151, 156, 158, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507125986Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172442024-02-13T21:46:11.815Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507147318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172452024-02-13T21:46:11.815Z{"msg":"[2]R flush_numbers[0..12]: [287, 297, 286, 297, 286, 264, 284, 295, 297, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507161863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172462024-02-13T21:46:11.815Z{"msg":"[2]R generation[0..12]: [153, 158, 152, 158, 152, 140, 151, 156, 158, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507182165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"Wait for a query_work_queue command to finish before sending IO
172472024-02-13T21:46:11.815Zdownstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172482024-02-13T21:46:11.815Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507199509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172492024-02-13T21:46:11.815Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50722412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172502024-02-13T21:46:11.815Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507258308Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172512024-02-13T21:46:11.815Z{"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507274532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172522024-02-13T21:46:11.815Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.50730575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"":"downstairs","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172532024-02-13T21:46:11.815Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507339728Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172542024-02-13T21:46:11.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507376995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"0","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172552024-02-13T21:46:11.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507415961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"1","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172562024-02-13T21:46:11.815Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507453407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"client":"2","":"downstairs","session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172572024-02-13T21:46:11.815Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507489664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172582024-02-13T21:46:11.816Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507525111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172592024-02-13T21:46:11.816Z{"msg":"3e5444dd-66c0-4c4a-af6b-fe1317a23aa2 is now active with session: f0d0735e-038d-4fd5-826c-cda882d7e5c5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507561198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172602024-02-13T21:46:11.816Z{"msg"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172612024-02-13T21:46:11.816Z:"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507601193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778,"session_id":"f0d0735e-038d-4fd5-826c-cda882d7e5c5"}
172622024-02-13T21:46:11.816Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:10.507630313Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
172632024-02-13T21:46:11.816ZLoading write count information from file "/tmp/test_repair_verify.data"
172642024-02-13T21:46:11.816ZRun Repair workload
172652024-02-13T21:46:11.816Z01/30 Write block 403 len 36864 data: 13 13 13 11 11 14 16 11 16
172662024-02-13T21:46:11.816Z02/30 Read block 401 len 16384
172672024-02-13T21:46:11.816Z03/30 Write block 21 len 16384 data: 13 15 13 13
172682024-02-13T21:46:11.816Z04/30 Read block 410 len 16384
172692024-02-13T21:46:11.816Z05/30 Write block 464 len 12288 data: 16 17 17
172702024-02-13T21:46:11.820Z06/30 Read block 499 len 20480
172712024-02-13T21:46:11.820Z07/30 Read block 577 len 32768
172722024-02-13T21:46:11.820Z08/30 Write block 38 len 12288 data: 12 12 13
172732024-02-13T21:46:11.820Z09/30 Read block 446 len 40960
172742024-02-13T21:46:11.820Z10/30 Read block 28 len 4096
172752024-02-13T21:46:11.820Z11/30 Read block 95 len 24576
172762024-02-13T21:46:11.820Z12/30 Write block 160 len 4096 data: 10
172772024-02-13T21:46:11.823Z13/30 Read block 571 len 32768
172782024-02-13T21:46:11.823Z14/30 Write block 109 len 12288 data: 8 7 10
172792024-02-13T21:46:11.823Z15/30 Write block 18 len 16384 data: 14 12 13 14
172802024-02-13T21:46:11.823Z16/30 Flush
172812024-02-13T21:46:11.826Z17/30 Write block 87 len 4096 data: 10
172822024-02-13T21:46:11.826Z18/30 Read block 124 len 4096
172832024-02-13T21:46:11.826Z19/30 Write block 320 len 12288 data: 18 17 18
172842024-02-13T21:46:11.827Z20/30 Flush
172852024-02-13T21:46:11.830Z21/30 Write block 378 len 4096 data: 13
172862024-02-13T21:46:11.830Z22/30 Write block 523 len 40960 data: 15 11 9 9 12 12 11 10 13 14
172872024-02-13T21:46:11.830Z23/30 Read block 245 len 28672
172882024-02-13T21:46:11.833Z24/30 Read block 355 len 28672
172892024-02-13T21:46:11.833Z25/30 Read block 76 len 20480
172902024-02-13T21:46:11.833Z26/30 Write block 552 len 8192 data: 15 14
172912024-02-13T21:46:11.837Z27/30 Read block 574 len 4096
172922024-02-13T21:46:11.837Z28/30 Read block 434 len 24576
172932024-02-13T21:46:11.837Z29/30 Write block 494 len 16384 data: 13 11 12 13
172942024-02-13T21:46:11.837Z30/30 Read block 304 len 4096
172952024-02-13T21:46:11.837Z----------------------------------------------------------------
172962024-02-13T21:46:11.837Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172972024-02-13T21:46:11.837ZStates: Active Active Active
172982024-02-13T21:46:11.837ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172992024-02-13T21:46:11.837Z 1 Acked 1000 Write 9 Done Done Done false
173002024-02-13T21:46:11.837Z 2 Acked 1001 Read 4 Done Done Done false
173012024-02-13T21:46:11.837Z 3 Acked 1002 Write 4 Done Done Done false
173022024-02-13T21:46:11.837Z 4 Acked 1003 Read 4 Done Done Done false
173032024-02-13T21:46:11.837Z 5 Acked 1004 Write 3 Done Done Done false
173042024-02-13T21:46:11.837Z 6 Acked 1005 Read 5 Done Done Sent false
173052024-02-13T21:46:11.837Z 7 Acked 1006 Read 8 Done Done Sent false
173062024-02-13T21:46:11.837Z 8 Acked 1007 Write 3 Done Done Sent false
173072024-02-13T21:46:11.837Z 9 Acked 1008 Read 10 Done Done Sent false
173082024-02-13T21:46:11.837Z 10 Acked 1009 Read 1 Done Done Sent false
173092024-02-13T21:46:11.837Z 11 Acked 1010 Read 6 Done Done Sent false
173102024-02-13T21:46:11.837Z 12 Acked 1011 Write 1 Done Done Sent false
173112024-02-13T21:46:11.837Z 13 Acked 1012 Read 8 Done Done Sent false
173122024-02-13T21:46:11.837Z 14 Acked 1013 Write 3 Done Done Sent false
173132024-02-13T21:46:11.837Z 15 Acked 1014 Write 4 Done Done Sent false
173142024-02-13T21:46:11.837Z 16 Acked 1015 Flush 0 Done Done Sent false
173152024-02-13T21:46:11.838Z 17 Acked 1016 Write 1 Done Done Sent false
173162024-02-13T21:46:11.838Z 18 Acked 1017 Read 1 Done Done Sent false
173172024-02-13T21:46:11.838Z 19 Acked 1018 Write 3 Done Done Sent false
173182024-02-13T21:46:11.838Z 20 Acked 1019 Flush 0 Done Done Sent false
173192024-02-13T21:46:11.838Z 21 Acked 1020 Write 1 Done Done Sent false
173202024-02-13T21:46:11.838Z 22 Acked 1021 Write 10 Done Done Sent false
173212024-02-13T21:46:11.838Z 23 Acked 1022 Read 7 Done Done Sent false
173222024-02-13T21:46:11.838Z 24 Acked 1023 Read 7 Done Done Sent false
173232024-02-13T21:46:11.838Z 25 Acked 1024 Read 5 Done Done Sent false
173242024-02-13T21:46:11.838Z 26 Acked 1025 Write 2 Done Done Sent false
173252024-02-13T21:46:11.838Z 27 Acked 1026 Read 1 Done Done Sent false
173262024-02-13T21:46:11.838Z 28 Acked 1027 Read 6 Done Done Sent false
173272024-02-13T21:46:11.838Z 29 Acked 1028 Write 4 Done Done Sent false
173282024-02-13T21:46:11.838Z 30 Acked 1029 Read 1 Sent Done Sent false
173292024-02-13T21:46:11.838Z STATES DS:0 DS:1 DS:2 TOTAL
173302024-02-13T21:46:11.838Z New 0 0 0 0
173312024-02-13T21:46:11.838Z Sent 1 0 25 26
173322024-02-13T21:46:11.838Z Done 29 30 5 64
173332024-02-13T21:46:11.838Z Skipped 0 0 0 0
173342024-02-13T21:46:11.838Z Error 0 0 0 0
173352024-02-13T21:46:11.838ZLast Flush: 1019 1019 0
173362024-02-13T21:46:11.838ZDownstairs last five completed:
173372024-02-13T21:46:11.838ZUpstairs last five completed: 30 29 28 27 26
173382024-02-13T21:46:11.844Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:10.535580044Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1778}
173392024-02-13T21:46:11.844ZWrote out file "/tmp/test_repair_verify.data"
173402024-02-13T21:46:11.844Z
173412024-02-13T21:46:11.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173422024-02-13T21:46:11.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173432024-02-13T21:46:11.860Z 0 000-019 160 160 153 299 299 287 F F F <---
173442024-02-13T21:46:11.860Z 1 020-039 160 160 158 299 299 297 F F T <---
173452024-02-13T21:46:11.860Z 2 040-059 160 160 152 299 299 286 F F F <---
173462024-02-13T21:46:11.860Z 3 060-079 158 158 158 297 297 297 F F F
173472024-02-13T21:46:11.860Z 4 080-099 160 160 152 300 300 286 F F F <---
173482024-02-13T21:46:11.860Z 5 100-119 160 160 140 299 299 264 F F F <---
173492024-02-13T21:46:11.860Z 6 120-139 151 151 151 284 284 284 F F F
173502024-02-13T21:46:11.863Z 7 140-159 156 156 156 295 295 295 F F F
173512024-02-13T21:46:11.863Z 8 160-179 160 160 158 299 299 297 F F F <---
173522024-02-13T21:46:11.863Z 9 180-199 157 157 157 296 296 296 F F F
173532024-02-13T21:46:11.863Z 10 200-219 148 148 148 280 280 280 F F F
173542024-02-13T21:46:11.864Z 11 220-239 159 159 159 298 298 298 F F F
173552024-02-13T21:46:11.864Z 12 240-259 158 158 158 297 297 297 F F F
173562024-02-13T21:46:11.864Z 13 260-279 159 159 159 298 298 298 F F F
173572024-02-13T21:46:11.864Z 14 280-299 159 159 159 298 298 298 F F F
173582024-02-13T21:46:11.864Z 15 300-319 158 158 158 297 297 297 F F F
173592024-02-13T21:46:11.864Z 16 320-339 160 160 148 300 300 280 F F F <---
173602024-02-13T21:46:11.864Z 17 340-359 152 152 152 286 286 286 F F F
173612024-02-13T21:46:11.864Z 18 360-379 152 152 152 286 286 286 T T F <---
173622024-02-13T21:46:11.864Z 19 380-399 154 154 154 291 291 291 F F F
173632024-02-13T21:46:11.864Z 20 400-419 160 160 154 299 299 289 F F T <---
173642024-02-13T21:46:11.864Z 21 420-439 155 155 155 292 292 292 F F F
173652024-02-13T21:46:11.864Z 22 440-459 156 156 156 294 294 294 F F F
173662024-02-13T21:46:11.864Z 23 460-479 160 160 158 299 299 297 F F T <---
173672024-02-13T21:46:11.864Z 24 480-499 158 158 158 297 297 297 T T F <---
173682024-02-13T21:46:11.864Z 25 500-519 158 158 158 297 297 297 F F F
173692024-02-13T21:46:11.864Z 26 520-539 157 157 157 296 296 296 T T F <---
173702024-02-13T21:46:11.864Z 27 540-559 156 156 156 293 293 293 T T F <---
173712024-02-13T21:46:11.864Z 28 560-579 157 157 157 296 296 296 F F F
173722024-02-13T21:46:11.864Z 29 580-599 154 154 154 290 290 290 F F F
173732024-02-13T21:46:11.864ZMax gen: 160, Max flush: 300
173742024-02-13T21:46:11.864ZError: Difference in extent metadata found!
173752024-02-13T21:46:11.864ZA Difference in extent metadata is expected here
173762024-02-13T21:46:11.864ZOn loop 80
173772024-02-13T21:46:11.864Z
173782024-02-13T21:46:11.868ZVerifying data now
173792024-02-13T21:46:17.498ZNo change after verify
173802024-02-13T21:46:17.498ZLoop: 80 Downstairs dump after verify (and repair):
173812024-02-13T21:46:17.513ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173822024-02-13T21:46:17.513Z 0 000-019 160 160 160 299 299 299 F F F
173832024-02-13T21:46:17.513Z 1 020-039 160 160 160 299 299 299 F F F
173842024-02-13T21:46:17.513Z 2 040-059 160 160 160 299 299 299 F F F
173852024-02-13T21:46:17.513Z 3 060-079 158 158 158 297 297 297 F F F
173862024-02-13T21:46:17.513Z 4 080-099 160 160 160 300 300 300 F F F
173872024-02-13T21:46:17.518Z 5 100-119 160 160 160 299 299 299 F F F
173882024-02-13T21:46:17.518Z 6 120-139 151 151 151 284 284 284 F F F
173892024-02-13T21:46:17.518Z 7 140-159 156 156 156 295 295 295 F F F
173902024-02-13T21:46:17.519Z 8 160-179 160 160 160 299 299 299 F F F
173912024-02-13T21:46:17.519Z 9 180-199 157 157 157 296 296 296 F F F
173922024-02-13T21:46:17.519Z 10 200-219 148 148 148 280 280 280 F F F
173932024-02-13T21:46:17.519Z 11 220-239 159 159 159 298 298 298 F F F
173942024-02-13T21:46:17.519Z 12 240-259 158 158 158 297 297 297 F F F
173952024-02-13T21:46:17.519Z 13 260-279 159 159 159 298 298 298 F F F
173962024-02-13T21:46:17.520Z 14 280-299 159 159 159 298 298 298 F F F
173972024-02-13T21:46:17.520Z 15 300-319 158 158 158 297 297 297 F F F
173982024-02-13T21:46:17.520Z 16 320-339 160 160 160 300 300 300 F F F
173992024-02-13T21:46:17.520Z 17 340-359 152 152 152 286 286 286 F F F
174002024-02-13T21:46:17.520Z 18 360-379 161 161 161 301 301 301 F F F
174012024-02-13T21:46:17.521Z 19 380-399 154 154 154 291 291 291 F F F
174022024-02-13T21:46:17.521Z 20 400-419 160 160 160 299 299 299 F F F
174032024-02-13T21:46:17.521Z 21 420-439 155 155 155 292 292 292 F F F
174042024-02-13T21:46:17.521Z 22 440-459 156 156 156 294 294 294 F F F
174052024-02-13T21:46:17.521Z 23 460-479 160 160 160 299 299 299 F F F
174062024-02-13T21:46:17.521Z 24 480-499 161 161 161 301 301 301 F F F
174072024-02-13T21:46:17.521Z 25 500-519 158 158 158 297 297 297 F F F
174082024-02-13T21:46:17.521Z 26 520-539 161 161 161 301 301 301 F F F
174092024-02-13T21:46:17.521Z 27 540-559 161 161 161 301 301 301 F F F
174102024-02-13T21:46:17.522Z 28 560-579 157 157 157 296 296 296 F F F
174112024-02-13T21:46:17.522Z 29 580-599 154 154 154 290 290 290 F F F
174122024-02-13T21:46:17.522ZMax gen: 161, Max flush: 301
174132024-02-13T21:46:17.522Z
174142024-02-13T21:46:17.522ZBegin loop 81
174152024-02-13T21:46:17.522ZDownstairs to restart: 1
174162024-02-13T21:46:17.533Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.229689215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
174172024-02-13T21:46:17.539Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
174182024-02-13T21:46:17.539Z2024-02-13T21:46:16.229954499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
174192024-02-13T21:46:17.540Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23003335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
174202024-02-13T21:46:17.540Z{"msg":"Crucible c5e6dcec-9c3e-4a5b-bb52-6fe8dce9b253 has session id: 560158eb-3aab-4e20-b364-d39d5961f02f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230076065Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174212024-02-13T21:46:17.540Z{"msg":"Upstairs opts: Upstairs UUID: c5e6dcec-9c3e-4a5b-bb52-6fe8dce9b253, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230122568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174222024-02-13T21:46:17.540Z{"msg":"Crucible stats registered with UUID: c5e6dcec-9c3e-4a5b-bb52-6fe8dce9b253","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230147069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174232024-02-13T21:46:17.540Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230167941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
174242024-02-13T21:46:17.540Z{"msg":"c5e6dcec-9c3e-4a5b-bb52-6fe8dce9b253 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230187984Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174252024-02-13T21:46:17.540Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230207687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174262024-02-13T21:46:17.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230230569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174272024-02-13T21:46:17.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230262957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174282024-02-13T21:46:17.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230285789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174292024-02-13T21:46:17.541Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23030925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174302024-02-13T21:46:17.541Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230361651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174312024-02-13T21:46:17.541Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230386412Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174322024-02-13T21:46:17.541Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23041974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174332024-02-13T21:46:17.541Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23044669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174342024-02-13T21:46:17.541Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.2305312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174352024-02-13T21:46:17.542Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230592387Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174362024-02-13T21:46:17.542Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.230636701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"io task","client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174372024-02-13T21:46:17.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.231297941Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174382024-02-13T21:46:17.542Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.231344254Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174392024-02-13T21:46:17.542Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.231389588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174402024-02-13T21:46:17.542Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.231430103Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174412024-02-13T21:46:17.543Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.231977865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174422024-02-13T21:46:17.543Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232013832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174432024-02-13T21:46:17.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232318321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174442024-02-13T21:46:17.543Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232357747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174452024-02-13T21:46:17.543Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232495407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174462024-02-13T21:46:17.543Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232528045Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174472024-02-13T21:46:17.543Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23254256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174482024-02-13T21:46:17.544Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232573099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174492024-02-13T21:46:17.544Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232600779Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174502024-02-13T21:46:17.544Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232614974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174512024-02-13T21:46:17.544Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.232844301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174522024-02-13T21:46:17.544Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233237478Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174532024-02-13T21:46:17.544Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233268856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174542024-02-13T21:46:17.544Z{"msg":"[0]R flush_numbers[0..12]: [299, 299, 299, 297, 300, 299, 284, 295, 299, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233284031Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174552024-02-13T21:46:17.544Z{"msg":"[0]R generation[0..12]: [160, 160, 160, 158, 160, 160, 151, 156, 160, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233305323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174562024-02-13T21:46:17.545Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233319998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174572024-02-13T21:46:17.545Z{"msg":"[1]R flush_numbers[0..12]: [299, 299, 299, 297, 300, 299, 284, 295, 299, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233336062Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174582024-02-13T21:46:17.545Z{"msg":"[1]R generation[0..12]: [160, 160, 160, 158, 160, 160, 151, 156, 160, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233350047Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174592024-02-13T21:46:17.545Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233371189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174602024-02-13T21:46:17.545Z{"msg":"[2]R flush_numbers[0..12]: [299, 299, 299, 297, 300, 299, 284, 295, 299, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233386134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174612024-02-13T21:46:17.545Z{"msg":"[2]R generation[0..12]: [160, 160, 160, 158, 160, 160, 151, 156, 160, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233406526Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
174622024-02-13T21:46:17.545Zpid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174632024-02-13T21:46:17.546Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233428808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174642024-02-13T21:46:17.546Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233456928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174652024-02-13T21:46:17.546Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233471233Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174662024-02-13T21:46:17.546Z{"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233495224Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174672024-02-13T21:46:17.546Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233518146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"":"downstairs","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174682024-02-13T21:46:17.546Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233547495Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174692024-02-13T21:46:17.546Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233569907Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"0","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174702024-02-13T21:46:17.547Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233597947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"1","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174712024-02-13T21:46:17.547Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233621568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"client":"2","":"downstairs","session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174722024-02-13T21:46:17.547Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233654587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174732024-02-13T21:46:17.547Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233677328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174742024-02-13T21:46:17.547Z{"msg":"c5e6dcec-9c3e-4a5b-bb52-6fe8dce9b253 is now active with session: 560158eb-3aab-4e20-b364-d39d5961f02f","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.233705818Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174752024-02-13T21:46:17.547Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23372732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786,"session_id":"560158eb-3aab-4e20-b364-d39d5961f02f"}
174762024-02-13T21:46:17.547ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174772024-02-13T21:46:17.547Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:16.23375587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
174782024-02-13T21:46:17.553ZLoading write count information from file "/tmp/test_repair_verify.data"
174792024-02-13T21:46:17.553ZRun Repair workload
174802024-02-13T21:46:17.553Z01/30 Write block 166 len 20480 data: 9 6 5 5 3
174812024-02-13T21:46:17.553Z02/30 Write block 545 len 28672 data: 13 12 10 8 12 13 18
174822024-02-13T21:46:17.553Z03/30 Read block 21 len 16384
174832024-02-13T21:46:17.553Z04/30 Read block 324 len 20480
174842024-02-13T21:46:17.554Z05/30 Read block 551 len 32768
174852024-02-13T21:46:17.554Z06/30 Write block 552 len 32768 data: 16 15 14 13 14 12 11 8
174862024-02-13T21:46:17.554Z07/30 Read block 106 len 32768
174872024-02-13T21:46:17.554Z08/30 Read block 195 len 20480
174882024-02-13T21:46:17.554Z09/30 Write block 386 len 12288 data: 14 13 12
174892024-02-13T21:46:17.554Z10/30 Write block 141 len 24576 data: 13 19 16 17 15 11
174902024-02-13T21:46:17.554Z11/30 Read block 176 len 32768
174912024-02-13T21:46:17.554Z12/30 Write block 250 len 40960 data: 14 16 13 11 10 12 14 14 11 10
174922024-02-13T21:46:17.555Z13/30 Read block 528 len 16384
174932024-02-13T21:46:17.555Z14/30 Read block 122 len 12288
174942024-02-13T21:46:17.555Z15/30 Flush
174952024-02-13T21:46:17.560Z16/30 Write block 88 len 40960 data: 10 10 9 10 9 8 8 9 10 11
174962024-02-13T21:46:17.560Z17/30 Read block 548 len 8192
174972024-02-13T21:46:17.560Z18/30 Write block 329 len 16384 data: 14 14 18 18
174982024-02-13T21:46:17.560Z19/30 Write block 257 len 32768 data: 15 12 11 7 7 8 9 10
174992024-02-13T21:46:17.560Z20/30 Read block 528 len 36864
175002024-02-13T21:46:17.564Z21/30 Write block 58 len 24576 data: 15 17 15 17 15 13
175012024-02-13T21:46:17.565Z22/30 Read block 482 len 20480
175022024-02-13T21:46:17.565Z23/30 Write block 157 len 16384 data: 10 11 10 11
175032024-02-13T21:46:17.565Z24/30 Write block 106 len 32768 data: 10 10 10 9 8 11 10 10
175042024-02-13T21:46:17.565Z25/30 Read block 99 len 36864
175052024-02-13T21:46:17.569Z26/30 Write block 67 len 32768 data: 12 13 17 16 14 16 14 15
175062024-02-13T21:46:17.569Z27/30 Write block 26 len 40960 data: 12 9 9 9 9 12 13 13 15 17
175072024-02-13T21:46:17.569Z28/30 Write block 163 len 40960 data: 11 9 7 10 7 6 6 4 5 9
175082024-02-13T21:46:17.570Z29/30 Write block 427 len 12288 data: 12 14 18
175092024-02-13T21:46:17.570Z30/30 Read block 588 len 40960
175102024-02-13T21:46:17.570Z----------------------------------------------------------------
175112024-02-13T21:46:17.570Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
175122024-02-13T21:46:17.570ZStates: Active Active Active
175132024-02-13T21:46:17.570ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175142024-02-13T21:46:17.570Z 1 Acked 1000 Write 5 Done Sent Done false
175152024-02-13T21:46:17.570Z 2 Acked 1001 Write 7 Done Sent Done false
175162024-02-13T21:46:17.570Z 3 Acked 1002 Read 4 Done Sent Done false
175172024-02-13T21:46:17.570Z 4 Acked 1003 Read 5 Done Sent Done false
175182024-02-13T21:46:17.570Z 5 Acked 1004 Read 8 Done Sent Done false
175192024-02-13T21:46:17.571Z 6 Acked 1005 Write 8 Done Sent Done false
175202024-02-13T21:46:17.571Z 7 Acked 1006 Read 8 Done Sent Done false
175212024-02-13T21:46:17.571Z 8 Acked 1007 Read 5 Done Sent Done false
175222024-02-13T21:46:17.571Z 9 Acked 1008 Write 3 Done Sent Done false
175232024-02-13T21:46:17.571Z 10 Acked 1009 Write 6 Done Sent Done false
175242024-02-13T21:46:17.571Z 11 Acked 1010 Read 8 Done Sent Done false
175252024-02-13T21:46:17.571Z 12 Acked 1011 Write 10 Done Sent Done false
175262024-02-13T21:46:17.571Z 13 Acked 1012 Read 4 Done Sent Done false
175272024-02-13T21:46:17.571Z 14 Acked 1013 Read 3 Done Sent Done false
175282024-02-13T21:46:17.571Z 15 Acked 1014 Flush 0 Done Sent Done false
175292024-02-13T21:46:17.571Z 16 Acked 1015 Write 10 Done Sent Done false
175302024-02-13T21:46:17.571Z 17 Acked 1016 Read 2 Done Sent Done false
175312024-02-13T21:46:17.571Z 18 Acked 1017 Write 4 Done Sent Done false
175322024-02-13T21:46:17.572Z 19 Acked 1018 Write 8 Done Sent Done false
175332024-02-13T21:46:17.572Z 20 Acked 1019 Read 9 Done Sent Done false
175342024-02-13T21:46:17.572Z 21 Acked 1020 Write 6 Done Sent Done false
175352024-02-13T21:46:17.572Z 22 Acked 1021 Read 5 Done Sent Done false
175362024-02-13T21:46:17.572Z 23 Acked 1022 Write 4 Done Sent Done false
175372024-02-13T21:46:17.572Z 24 Acked 1023 Write 8 Done Sent Done false
175382024-02-13T21:46:17.572Z 25 Acked 1024 Read 9 Done Sent Done false
175392024-02-13T21:46:17.572Z 26 Acked 1025 Write 8 Done Sent Done false
175402024-02-13T21:46:17.572Z 27 Acked 1026 Write 10 Done Sent Done false
175412024-02-13T21:46:17.572Z 28 Acked 1027 Write 10 Done Sent Done false
175422024-02-13T21:46:17.572Z 29 Acked 1028 Write 3 Done Sent Done false
175432024-02-13T21:46:17.572Z 30 Acked 1029 Read 10 Sent Sent Done false
175442024-02-13T21:46:17.572Z STATES DS:0 DS:1 DS:2 TOTAL
175452024-02-13T21:46:17.572Z New 0 0 0 0
175462024-02-13T21:46:17.573Z Sent 1 30 0 31
175472024-02-13T21:46:17.573Z Done 29 0 30 59
175482024-02-13T21:46:17.573Z Skipped 0 0 0 0
175492024-02-13T21:46:17.573Z Error 0 0 0 0
175502024-02-13T21:46:17.573ZLast Flush: 1014 0 1014
175512024-02-13T21:46:17.573ZDownstairs last five completed:
175522024-02-13T21:46:17.573ZUpstairs last five completed: 30 29 28 27 26
175532024-02-13T21:46:17.573Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:16.265468516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1786}
175542024-02-13T21:46:17.578ZWrote out file "/tmp/test_repair_verify.data"
175552024-02-13T21:46:17.578Z
175562024-02-13T21:46:17.578ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175572024-02-13T21:46:17.594ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175582024-02-13T21:46:17.594Z 0 000-019 160 160 160 299 299 299 F F F
175592024-02-13T21:46:17.594Z 1 020-039 160 160 160 299 299 299 T F T <---
175602024-02-13T21:46:17.594Z 2 040-059 160 160 160 299 299 299 T F T <---
175612024-02-13T21:46:17.594Z 3 060-079 158 158 158 297 297 297 T F T <---
175622024-02-13T21:46:17.594Z 4 080-099 160 160 160 300 300 300 T F T <---
175632024-02-13T21:46:17.594Z 5 100-119 160 160 160 299 299 299 T F T <---
175642024-02-13T21:46:17.594Z 6 120-139 151 151 151 284 284 284 F F F
175652024-02-13T21:46:17.594Z 7 140-159 162 156 162 302 295 302 T F T <---
175662024-02-13T21:46:17.594Z 8 160-179 162 160 162 302 299 302 T F T <---
175672024-02-13T21:46:17.594Z 9 180-199 157 157 157 296 296 296 F F F
175682024-02-13T21:46:17.594Z 10 200-219 148 148 148 280 280 280 F F F
175692024-02-13T21:46:17.598Z 11 220-239 159 159 159 298 298 298 F F F
175702024-02-13T21:46:17.598Z 12 240-259 162 158 162 302 297 302 T F T <---
175712024-02-13T21:46:17.598Z 13 260-279 159 159 159 298 298 298 T F T <---
175722024-02-13T21:46:17.598Z 14 280-299 159 159 159 298 298 298 F F F
175732024-02-13T21:46:17.598Z 15 300-319 158 158 158 297 297 297 F F F
175742024-02-13T21:46:17.598Z 16 320-339 160 160 160 300 300 300 T F T <---
175752024-02-13T21:46:17.598Z 17 340-359 152 152 152 286 286 286 F F F
175762024-02-13T21:46:17.598Z 18 360-379 161 161 161 301 301 301 F F F
175772024-02-13T21:46:17.598Z 19 380-399 162 154 162 302 291 302 F F F <---
175782024-02-13T21:46:17.598Z 20 400-419 160 160 160 299 299 299 F F F
175792024-02-13T21:46:17.598Z 21 420-439 155 155 155 292 292 292 T F T <---
175802024-02-13T21:46:17.598Z 22 440-459 156 156 156 294 294 294 F F F
175812024-02-13T21:46:17.599Z 23 460-479 160 160 160 299 299 299 F F F
175822024-02-13T21:46:17.599Z 24 480-499 161 161 161 301 301 301 F F F
175832024-02-13T21:46:17.599Z 25 500-519 158 158 158 297 297 297 F F F
175842024-02-13T21:46:17.599Z 26 520-539 161 161 161 301 301 301 F F F
175852024-02-13T21:46:17.599Z 27 540-559 162 161 162 302 301 302 F F F <---
175862024-02-13T21:46:17.599Z 28 560-579 157 157 157 296 296 296 F F F
175872024-02-13T21:46:17.599Z 29 580-599 154 154 154 290 290 290 F F F
175882024-02-13T21:46:17.599ZMax gen: 162, Max flush: 302
175892024-02-13T21:46:17.599ZError: Difference in extent metadata found!
175902024-02-13T21:46:17.599ZA Difference in extent metadata is expected here
175912024-02-13T21:46:17.599ZOn loop 81
175922024-02-13T21:46:17.599Z
175932024-02-13T21:46:17.602ZVerifying data now
175942024-02-13T21:46:22.251ZNo change after verify
175952024-02-13T21:46:22.251ZLoop: 81 Downstairs dump after verify (and repair):
175962024-02-13T21:46:22.266ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175972024-02-13T21:46:22.266Z 0 000-019 160 160 160 299 299 299 F F F
175982024-02-13T21:46:22.266Z 1 020-039 163 163 163 303 303 303 F F F
175992024-02-13T21:46:22.267Z 2 040-059 163 163 163 303 303 303 F F F
176002024-02-13T21:46:22.267Z 3 060-079 163 163 163 303 303 303 F F F
176012024-02-13T21:46:22.267Z 4 080-099 163 163 163 303 303 303 F F F
176022024-02-13T21:46:22.267Z 5 100-119 163 163 163 303 303 303 F F F
176032024-02-13T21:46:22.267Z 6 120-139 151 151 151 284 284 284 F F F
176042024-02-13T21:46:22.271Z 7 140-159 163 163 163 303 303 303 F F F
176052024-02-13T21:46:22.271Z 8 160-179 163 163 163 303 303 303 F F F
176062024-02-13T21:46:22.271Z 9 180-199 157 157 157 296 296 296 F F F
176072024-02-13T21:46:22.271Z 10 200-219 148 148 148 280 280 280 F F F
176082024-02-13T21:46:22.271Z 11 220-239 159 159 159 298 298 298 F F F
176092024-02-13T21:46:22.271Z 12 240-259 163 163 163 303 303 303 F F F
176102024-02-13T21:46:22.271Z 13 260-279 163 163 163 303 303 303 F F F
176112024-02-13T21:46:22.271Z 14 280-299 159 159 159 298 298 298 F F F
176122024-02-13T21:46:22.271Z 15 300-319 158 158 158 297 297 297 F F F
176132024-02-13T21:46:22.271Z 16 320-339 163 163 163 303 303 303 F F F
176142024-02-13T21:46:22.271Z 17 340-359 152 152 152 286 286 286 F F F
176152024-02-13T21:46:22.271Z 18 360-379 161 161 161 301 301 301 F F F
176162024-02-13T21:46:22.271Z 19 380-399 162 162 162 302 302 302 F F F
176172024-02-13T21:46:22.272Z 20 400-419 160 160 160 299 299 299 F F F
176182024-02-13T21:46:22.272Z 21 420-439 163 163 163 303 303 303 F F F
176192024-02-13T21:46:22.272Z 22 440-459 156 156 156 294 294 294 F F F
176202024-02-13T21:46:22.272Z 23 460-479 160 160 160 299 299 299 F F F
176212024-02-13T21:46:22.272Z 24 480-499 161 161 161 301 301 301 F F F
176222024-02-13T21:46:22.272Z 25 500-519 158 158 158 297 297 297 F F F
176232024-02-13T21:46:22.272Z 26 520-539 161 161 161 301 301 301 F F F
176242024-02-13T21:46:22.272Z 27 540-559 162 162 162 302 302 302 F F F
176252024-02-13T21:46:22.272Z 28 560-579 157 157 157 296 296 296 F F F
176262024-02-13T21:46:22.272Z 29 580-599 154 154 154 290 290 290 F F F
176272024-02-13T21:46:22.272ZMax gen: 163, Max flush: 303
176282024-02-13T21:46:22.272Z
176292024-02-13T21:46:22.272ZBegin loop 82
176302024-02-13T21:46:22.272ZDownstairs to restart: 0
176312024-02-13T21:46:22.287Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.982633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
176322024-02-13T21:46:22.292Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.982968658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
176332024-02-13T21:46:22.292Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983043991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
176342024-02-13T21:46:22.292Z{"msg":"Crucible 3d16c92d-f65c-47aa-9666-193f981da429 has session id: cf0aded4-7033-494d-a501-7573c8605777","v":0,"name":"crucible","level":30Crucible runtime is spawned
176352024-02-13T21:46:22.292Z,"time":"2024-02-13T21:46:20.983080527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176362024-02-13T21:46:22.292Z{"msg":"Upstairs opts: Upstairs UUID: 3d16c92d-f65c-47aa-9666-193f981da429, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983122692Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176372024-02-13T21:46:22.292Z{"msg":"Crucible stats registered with UUID: 3d16c92d-f65c-47aa-9666-193f981da429","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983149982Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176382024-02-13T21:46:22.292Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983229863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
176392024-02-13T21:46:22.293Z{"msg":"3d16c92d-f65c-47aa-9666-193f981da429 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983334275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176402024-02-13T21:46:22.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983368823Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176412024-02-13T21:46:22.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983385697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176422024-02-13T21:46:22.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983412487Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176432024-02-13T21:46:22.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983426882Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176442024-02-13T21:46:22.293Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983440557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176452024-02-13T21:46:22.293Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983457271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176462024-02-13T21:46:22.293Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983484891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176472024-02-13T21:46:22.293Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983506893Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176482024-02-13T21:46:22.293Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.98356976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176492024-02-13T21:46:22.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983665575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176502024-02-13T21:46:22.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983698863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176512024-02-13T21:46:22.293Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.983726773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"io task","client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176522024-02-13T21:46:22.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.9843967Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176532024-02-13T21:46:22.293Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.984440584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176542024-02-13T21:46:22.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.984476351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176552024-02-13T21:46:22.293Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.984504661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176562024-02-13T21:46:22.293Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.984992614Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176572024-02-13T21:46:22.293Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985102934Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176582024-02-13T21:46:22.293Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985272372Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176592024-02-13T21:46:22.293Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985311488Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176602024-02-13T21:46:22.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985434424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176612024-02-13T21:46:22.293Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985488154Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176622024-02-13T21:46:22.293Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985523291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176632024-02-13T21:46:22.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985544744Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176642024-02-13T21:46:22.294Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985576422Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176652024-02-13T21:46:22.294Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.98560866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176662024-02-13T21:46:22.294Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.985907912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176672024-02-13T21:46:22.298Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986342194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176682024-02-13T21:46:22.298Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986383699Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176692024-02-13T21:46:22.298Z{"msg":"[0]R flush_numbers[0..12]: [299, 303, 303, 303, 303, 303, 284, 303, 303, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986402532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176702024-02-13T21:46:22.298Z{"msg":"[0]R generation[0..12]: [160, 163, 163, 163, 163, 163, 151, 163, 163, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986430522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176712024-02-13T21:46:22.298Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986448716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176722024-02-13T21:46:22.298Z{"msg":"[1]R flush_numbers[0..12]: [299, 303, 303, 303, 303, 303, 284, 303, 303, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986473716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176732024-02-13T21:46:22.298Z{"msg":"[1]R generation[0..12]: [160, 163, 163, 163, 163, 163, 151, 163, 163, 157, 148, 159]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
176742024-02-13T21:46:22.298Z"time":"2024-02-13T21:46:20.986488761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176752024-02-13T21:46:22.298Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986507104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176762024-02-13T21:46:22.298Z{"msg":"[2]R flush_numbers[0..12]: [299, 303, 303, 303, 303, 303, 284, 303, 303, 296, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986521129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176772024-02-13T21:46:22.298Z{"msg":"[2]R generation[0..12]: [160, 163, 163, 163, 163, 163, 151, 163, 163, 157, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986538703Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176782024-02-13T21:46:22.298Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.98657438Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176792024-02-13T21:46:22.298Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986603969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176802024-02-13T21:46:22.298Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986617864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176812024-02-13T21:46:22.298Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986646364Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176822024-02-13T21:46:22.298Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986662288Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"":"downstairs","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176832024-02-13T21:46:22.298Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986690928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176842024-02-13T21:46:22.298Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986705243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"0","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176852024-02-13T21:46:22.298Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986718588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"1","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176862024-02-13T21:46:22.298Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986731813Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"client":"2","":"downstairs","session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176872024-02-13T21:46:22.298Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986755914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176882024-02-13T21:46:22.298Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986778376Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176892024-02-13T21:46:22.298Z{"msg":"3d16c92d-f65c-47aa-9666-193f981da429 is now active with session: cf0aded4-7033-494d-a501-7573c8605777","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986803107Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176902024-02-13T21:46:22.299Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986837724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794,"session_id":"cf0aded4-7033-494d-a501-7573c8605777"}
176912024-02-13T21:46:22.299Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:20.986875761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
176922024-02-13T21:46:22.299ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176932024-02-13T21:46:22.299ZLoading write count information from file "/tmp/test_repair_verify.data"
176942024-02-13T21:46:22.299ZRun Repair workload
176952024-02-13T21:46:22.299Z01/30 Write block 482 len 32768 data: 13 12 11 10 11 10 11 11
176962024-02-13T21:46:22.299Z02/30 Read block 82 len 36864
176972024-02-13T21:46:22.299Z03/30 Read block 442 len 8192
176982024-02-13T21:46:22.299Z04/30 Write block 370 len 12288 data: 13 15 19
176992024-02-13T21:46:22.299Z05/30 Flush
177002024-02-13T21:46:22.309Z06/30 Write block 88 len 24576 data: 11 11 10 11 10 9
177012024-02-13T21:46:22.309Z07/30 Read block 583 len 4096
177022024-02-13T21:46:22.309Z08/30 Read block 233 len 12288
177032024-02-13T21:46:22.309Z09/30 Read block 2 len 36864
177042024-02-13T21:46:22.309Z10/30 Write block 185 len 12288 data: 15 12 11
177052024-02-13T21:46:22.309Z11/30 Read block 142 len 28672
177062024-02-13T21:46:22.309Z12/30 Read block 540 len 40960
177072024-02-13T21:46:22.309Z13/30 Read block 261 len 28672
177082024-02-13T21:46:22.309Z14/30 Read block 582 len 28672
177092024-02-13T21:46:22.309Z15/30 Read block 450 len 28672
177102024-02-13T21:46:22.309Z16/30 Read block 47 len 24576
177112024-02-13T21:46:22.313Z17/30 Read block 373 len 4096
177122024-02-13T21:46:22.313Z18/30 Read block 479 len 24576
177132024-02-13T21:46:22.313Z19/30 Read block 164 len 40960
177142024-02-13T21:46:22.313Z20/30 Write block 326 len 24576 data: 17 15 15 15 15 19
177152024-02-13T21:46:22.313Z21/30 Write block 330 len 12288 data: 16 20 19
177162024-02-13T21:46:22.313Z22/30 Read block 525 len 32768
177172024-02-13T21:46:22.316Z23/30 Read block 11 len 8192
177182024-02-13T21:46:22.316Z24/30 Read block 225 len 16384
177192024-02-13T21:46:22.316Z25/30 Read block 212 len 24576
177202024-02-13T21:46:22.319Z26/30 Read block 568 len 8192
177212024-02-13T21:46:22.319Z27/30 Write block 498 len 20480 data: 12 10 11 7 9
177222024-02-13T21:46:22.319Z28/30 Read block 199 len 24576
177232024-02-13T21:46:22.319Z29/30 Read block 65 len 12288
177242024-02-13T21:46:22.323Z30/30 Write block 30 len 24576 data: 10 13 14 14 16 18
177252024-02-13T21:46:22.323Z----------------------------------------------------------------
177262024-02-13T21:46:22.323Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177272024-02-13T21:46:22.323ZStates: Active Active Active
177282024-02-13T21:46:22.323ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177292024-02-13T21:46:22.323Z 1 Acked 1000 Write 8 Done Done Done false
177302024-02-13T21:46:22.323Z 2 Acked 1001 Read 9 Done Done Done false
177312024-02-13T21:46:22.323Z 3 Acked 1002 Read 2 Sent Done Done false
177322024-02-13T21:46:22.323Z 4 Acked 1003 Write 3 Sent Done Done false
177332024-02-13T21:46:22.323Z 5 Acked 1004 Flush 0 Sent Done Done false
177342024-02-13T21:46:22.323Z 6 Acked 1005 Write 6 Sent Done Done false
177352024-02-13T21:46:22.323Z 7 Acked 1006 Read 1 Sent Done Done false
177362024-02-13T21:46:22.323Z 8 Acked 1007 Read 3 Sent Done Done false
177372024-02-13T21:46:22.323Z 9 Acked 1008 Read 9 Sent Done Done false
177382024-02-13T21:46:22.323Z 10 Acked 1009 Write 3 Sent Done Done false
177392024-02-13T21:46:22.323Z 11 Acked 1010 Read 7 Sent Done Done false
177402024-02-13T21:46:22.323Z 12 Acked 1011 Read 10 Sent Done Done false
177412024-02-13T21:46:22.323Z 13 Acked 1012 Read 7 Sent Done Done false
177422024-02-13T21:46:22.323Z 14 Acked 1013 Read 7 Sent Done Done false
177432024-02-13T21:46:22.323Z 15 Acked 1014 Read 7 Sent Done Done false
177442024-02-13T21:46:22.323Z 16 Acked 1015 Read 6 Sent Done Done false
177452024-02-13T21:46:22.323Z 17 Acked 1016 Read 1 Sent Done Done false
177462024-02-13T21:46:22.323Z 18 Acked 1017 Read 6 Sent Done Done false
177472024-02-13T21:46:22.324Z 19 Acked 1018 Read 10 Sent Done Done false
177482024-02-13T21:46:22.324Z 20 Acked 1019 Write 6 Sent Done Done false
177492024-02-13T21:46:22.324Z 21 Acked 1020 Write 3 Sent Done Done false
177502024-02-13T21:46:22.324Z 22 Acked 1021 Read 8 Sent Done Done false
177512024-02-13T21:46:22.324Z 23 Acked 1022 Read 2 Sent Done Done false
177522024-02-13T21:46:22.324Z 24 Acked 1023 Read 4 Sent Done Done false
177532024-02-13T21:46:22.324Z 25 Acked 1024 Read 6 Sent Done Done false
177542024-02-13T21:46:22.324Z 26 Acked 1025 Read 2 Sent Done Done false
177552024-02-13T21:46:22.324Z 27 Acked 1026 Write 5 Sent Done Done false
177562024-02-13T21:46:22.324Z 28 Acked 1027 Read 6 Sent Done Done false
177572024-02-13T21:46:22.324Z 29 Acked 1028 Read 3 Sent Done Done false
177582024-02-13T21:46:22.324Z 30 Acked 1029 Write 6 Sent Sent Sent false
177592024-02-13T21:46:22.324Z STATES DS:0 DS:1 DS:2 TOTAL
177602024-02-13T21:46:22.324Z New 0 0 0 0
177612024-02-13T21:46:22.324Z Sent 28 1 1 30
177622024-02-13T21:46:22.324Z Done 2 29 29 60
177632024-02-13T21:46:22.324Z Skipped 0 0 0 0
177642024-02-13T21:46:22.324Z Error 0 0 0 0
177652024-02-13T21:46:22.324ZLast Flush: 0 1004 1004
177662024-02-13T21:46:22.324ZDownstairs last five completed:
177672024-02-13T21:46:22.324ZUpstairs last five completed: 30 29 28 27 26
177682024-02-13T21:46:22.324Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:21.015627661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1794}
177692024-02-13T21:46:22.324ZWrote out file "/tmp/test_repair_verify.data"
177702024-02-13T21:46:22.331Z
177712024-02-13T21:46:22.331ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177722024-02-13T21:46:22.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177732024-02-13T21:46:22.346Z 0 000-019 160 160 160 299 299 299 F F F
177742024-02-13T21:46:22.346Z 1 020-039 163 163 163 303 303 303 F T T <---
177752024-02-13T21:46:22.346Z 2 040-059 163 163 163 303 303 303 F F F
177762024-02-13T21:46:22.346Z 3 060-079 163 163 163 303 303 303 F F F
177772024-02-13T21:46:22.346Z 4 080-099 163 163 163 303 303 303 F T T <---
177782024-02-13T21:46:22.346Z 5 100-119 163 163 163 303 303 303 F F F
177792024-02-13T21:46:22.346Z 6 120-139 151 151 151 284 284 284 F F F
177802024-02-13T21:46:22.346Z 7 140-159 163 163 163 303 303 303 F F F
177812024-02-13T21:46:22.346Z 8 160-179 163 163 163 303 303 303 F F F
177822024-02-13T21:46:22.349Z 9 180-199 157 157 157 296 296 296 F T T <---
177832024-02-13T21:46:22.349Z 10 200-219 148 148 148 280 280 280 F F F
177842024-02-13T21:46:22.350Z 11 220-239 159 159 159 298 298 298 F F F
177852024-02-13T21:46:22.350Z 12 240-259 163 163 163 303 303 303 F F F
177862024-02-13T21:46:22.350Z 13 260-279 163 163 163 303 303 303 F F F
177872024-02-13T21:46:22.350Z 14 280-299 159 159 159 298 298 298 F F F
177882024-02-13T21:46:22.350Z 15 300-319 158 158 158 297 297 297 F F F
177892024-02-13T21:46:22.350Z 16 320-339 163 163 163 303 303 303 F T T <---
177902024-02-13T21:46:22.350Z 17 340-359 152 152 152 286 286 286 F F F
177912024-02-13T21:46:22.350Z 18 360-379 161 164 164 301 304 304 F F F <---
177922024-02-13T21:46:22.350Z 19 380-399 162 162 162 302 302 302 F F F
177932024-02-13T21:46:22.350Z 20 400-419 160 160 160 299 299 299 F F F
177942024-02-13T21:46:22.350Z 21 420-439 163 163 163 303 303 303 F F F
177952024-02-13T21:46:22.350Z 22 440-459 156 156 156 294 294 294 F F F
177962024-02-13T21:46:22.350Z 23 460-479 160 160 160 299 299 299 F F F
177972024-02-13T21:46:22.350Z 24 480-499 161 164 164 301 304 304 T T T <---
177982024-02-13T21:46:22.350Z 25 500-519 158 158 158 297 297 297 F T T <---
177992024-02-13T21:46:22.350Z 26 520-539 161 161 161 301 301 301 F F F
178002024-02-13T21:46:22.350Z 27 540-559 162 162 162 302 302 302 F F F
178012024-02-13T21:46:22.350Z 28 560-579 157 157 157 296 296 296 F F F
178022024-02-13T21:46:22.350Z 29 580-599 154 154 154 290 290 290 F F F
178032024-02-13T21:46:22.350ZMax gen: 164, Max flush: 304
178042024-02-13T21:46:22.350ZError: Difference in extent metadata found!
178052024-02-13T21:46:22.350ZA Difference in extent metadata is expected here
178062024-02-13T21:46:22.350ZOn loop 82
178072024-02-13T21:46:22.351Z
178082024-02-13T21:46:22.354ZVerifying data now
178092024-02-13T21:46:26.762ZNo change after verify
178102024-02-13T21:46:26.762ZLoop: 82 Downstairs dump after verify (and repair):
178112024-02-13T21:46:26.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178122024-02-13T21:46:26.773Z 0 000-019 160 160 160 299 299 299 F F F
178132024-02-13T21:46:26.773Z 1 020-039 165 165 165 305 305 305 F F F
178142024-02-13T21:46:26.774Z 2 040-059 163 163 163 303 303 303 F F F
178152024-02-13T21:46:26.774Z 3 060-079 163 163 163 303 303 303 F F F
178162024-02-13T21:46:26.774Z 4 080-099 165 165 165 305 305 305 F F F
178172024-02-13T21:46:26.774Z 5 100-119 163 163 163 303 303 303 F F F
178182024-02-13T21:46:26.774Z 6 120-139 151 151 151 284 284 284 F F F
178192024-02-13T21:46:26.774Z 7 140-159 163 163 163 303 303 303 F F F
178202024-02-13T21:46:26.777Z 8 160-179 163 163 163 303 303 303 F F F
178212024-02-13T21:46:26.777Z 9 180-199 165 165 165 305 305 305 F F F
178222024-02-13T21:46:26.777Z 10 200-219 148 148 148 280 280 280 F F F
178232024-02-13T21:46:26.777Z 11 220-239 159 159 159 298 298 298 F F F
178242024-02-13T21:46:26.777Z 12 240-259 163 163 163 303 303 303 F F F
178252024-02-13T21:46:26.777Z 13 260-279 163 163 163 303 303 303 F F F
178262024-02-13T21:46:26.777Z 14 280-299 159 159 159 298 298 298 F F F
178272024-02-13T21:46:26.777Z 15 300-319 158 158 158 297 297 297 F F F
178282024-02-13T21:46:26.777Z 16 320-339 165 165 165 305 305 305 F F F
178292024-02-13T21:46:26.777Z 17 340-359 152 152 152 286 286 286 F F F
178302024-02-13T21:46:26.777Z 18 360-379 164 164 164 304 304 304 F F F
178312024-02-13T21:46:26.777Z 19 380-399 162 162 162 302 302 302 F F F
178322024-02-13T21:46:26.777Z 20 400-419 160 160 160 299 299 299 F F F
178332024-02-13T21:46:26.777Z 21 420-439 163 163 163 303 303 303 F F F
178342024-02-13T21:46:26.777Z 22 440-459 156 156 156 294 294 294 F F F
178352024-02-13T21:46:26.777Z 23 460-479 160 160 160 299 299 299 F F F
178362024-02-13T21:46:26.777Z 24 480-499 165 165 165 305 305 305 F F F
178372024-02-13T21:46:26.778Z 25 500-519 165 165 165 305 305 305 F F F
178382024-02-13T21:46:26.778Z 26 520-539 161 161 161 301 301 301 F F F
178392024-02-13T21:46:26.778Z 27 540-559 162 162 162 302 302 302 F F F
178402024-02-13T21:46:26.778Z 28 560-579 157 157 157 296 296 296 F F F
178412024-02-13T21:46:26.778Z 29 580-599 154 154 154 290 290 290 F F F
178422024-02-13T21:46:26.778ZMax gen: 165, Max flush: 305
178432024-02-13T21:46:26.778Z
178442024-02-13T21:46:26.778ZBegin loop 83
178452024-02-13T21:46:26.778ZDownstairs to restart: 1
178462024-02-13T21:46:26.795Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.488663951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
178472024-02-13T21:46:26.795Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.488916079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
178482024-02-13T21:46:26.795Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.488933493Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
178492024-02-13T21:46:26.795Z{"msg":"Crucible 6e988335-835a-4cbe-b44f-7c96f3642c5c has session id: b1158f7b-0fb1-4710-a540-b0d3d3821ee7","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.488951706Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178502024-02-13T21:46:26.800Z{"msg":"Upstairs opts: Upstairs UUID: 6e988335-835a-4cbe-b44f-7c96f3642c5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.488999429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178512024-02-13T21:46:26.800ZCrucible runtime is spawned
178522024-02-13T21:46:26.800Z{"msg":"Crucible stats registered with UUID: 6e988335-835a-4cbe-b44f-7c96f3642c5c","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489069354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178532024-02-13T21:46:26.800Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489126513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
178542024-02-13T21:46:26.800Z{"msg":"6e988335-835a-4cbe-b44f-7c96f3642c5c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489201956Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178552024-02-13T21:46:26.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489237443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178562024-02-13T21:46:26.800Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489254057Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178572024-02-13T21:46:26.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.48927322Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178582024-02-13T21:46:26.800Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489295492Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178592024-02-13T21:46:26.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489318244Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178602024-02-13T21:46:26.800Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489362077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178612024-02-13T21:46:26.800Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489386229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178622024-02-13T21:46:26.800Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489415038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178632024-02-13T21:46:26.800Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489451415Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178642024-02-13T21:46:26.800Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.48957593Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178652024-02-13T21:46:26.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489615976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178662024-02-13T21:46:26.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.489644575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"io task","client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178672024-02-13T21:46:26.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.490331246Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178682024-02-13T21:46:26.801Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.490368323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178692024-02-13T21:46:26.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.490384007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178702024-02-13T21:46:26.801Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.490464418Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178712024-02-13T21:46:26.801Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.490974263Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178722024-02-13T21:46:26.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491024145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178732024-02-13T21:46:26.801Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491049865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178742024-02-13T21:46:26.801Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491076966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178752024-02-13T21:46:26.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491598416Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178762024-02-13T21:46:26.801Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491634763Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178772024-02-13T21:46:26.801Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491651187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178782024-02-13T21:46:26.801Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491665932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178792024-02-13T21:46:26.801Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491688424Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178802024-02-13T21:46:26.801Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491702189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178812024-02-13T21:46:26.801Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.491715524Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178822024-02-13T21:46:26.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492178116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178832024-02-13T21:46:26.805Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492225139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178842024-02-13T21:46:26.805Z{"msg":"[0]R flush_numbers[0..12]: [299, 305, 303, 303, 305, 303, 284, 303, 303, 305, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492243212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178852024-02-13T21:46:26.805Z{"msg":"[0]R generation[0..12]: [160, 165, 163, 163, 165, 163, 151, 163, 163, 165, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492258797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178862024-02-13T21:46:26.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492273911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178872024-02-13T21:46:26.805Z{"msg":"[1]R flush_numbers[0..12]: [299, 305, 303, 303, 305, 303, 284, 303, 303, 305, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492288316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178882024-02-13T21:46:26.805Z{"msg":"[1]R generation[0..12]: [160, 165, 163, 163, 165, 163, 151, 163, 163, 165, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492315756Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178892024-02-13T21:46:26.805Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.49233423Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178902024-02-13T21:46:26.805Z{"msg":"[2]R flush_numbers[0..12]: [299, 305, 303, 303, 305, 303, 284, 303, 303, 305, 280, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492362959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178912024-02-13T21:46:26.805Z{"msg":"[2]R generation[0..12]: [160, 165, 163, 163, 165, 163, 151, 163, 163, 165, 148, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492377963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178922024-02-13T21:46:26.805Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
178932024-02-13T21:46:26.805Z,"time":"2024-02-13T21:46:25.492392168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178942024-02-13T21:46:26.806Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.49241513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178952024-02-13T21:46:26.806Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492428675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178962024-02-13T21:46:26.806Z{"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492441651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178972024-02-13T21:46:26.806Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492462703Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"":"downstairs","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178982024-02-13T21:46:26.806Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492477787Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
178992024-02-13T21:46:26.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492490453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"0","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179002024-02-13T21:46:26.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492504278Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"1","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179012024-02-13T21:46:26.806Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492517513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"client":"2","":"downstairs","session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179022024-02-13T21:46:26.806Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492531528Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179032024-02-13T21:46:26.806Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492543993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179042024-02-13T21:46:26.806Z{"msg":"6e988335-835a-4cbe-b44f-7c96f3642c5c is now active with session: b1158f7b-0fb1-4710-a540-b0d3d3821ee7","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492564426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179052024-02-13T21:46:26.806Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492576651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802,"session_id":"b1158f7b-0fb1-4710-a540-b0d3d3821ee7"}
179062024-02-13T21:46:26.806Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:25.492589517Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
179072024-02-13T21:46:26.806ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179082024-02-13T21:46:26.806ZLoading write count information from file "/tmp/test_repair_verify.data"
179092024-02-13T21:46:26.806ZRun Repair workload
179102024-02-13T21:46:26.806Z01/30 Write block 69 len 40960 data: 18 17 15 17 15 16 13 13 11 7
179112024-02-13T21:46:26.806Z02/30 Read block 109 len 20480
179122024-02-13T21:46:26.806Z03/30 Write block 388 len 36864 data: 13 13 11 9 9 11 12 10 10
179132024-02-13T21:46:26.806Z04/30 Read block 383 len 28672
179142024-02-13T21:46:26.809Z05/30 Read block 518 len 16384
179152024-02-13T21:46:26.810Z06/30 Flush
179162024-02-13T21:46:26.810Z07/30 Write block 161 len 8192 data: 12 12
179172024-02-13T21:46:26.810Z08/30 Read block 321 len 4096
179182024-02-13T21:46:26.810Z09/30 Read block 313 len 16384
179192024-02-13T21:46:26.813Z10/30 Write block 453 len 8192 data: 13 13
179202024-02-13T21:46:26.813Z11/30 Write block 206 len 20480 data: 4 5 9 10 10
179212024-02-13T21:46:26.813Z12/30 Write block 384 len 8192 data: 13 13
179222024-02-13T21:46:26.813Z13/30 Write block 557 len 4096 data: 13
179232024-02-13T21:46:26.813Z14/30 Write block 59 len 20480 data: 18 16 18 16 14
179242024-02-13T21:46:26.816Z15/30 Read block 101 len 32768
179252024-02-13T21:46:26.816Z16/30 Read block 480 len 36864
179262024-02-13T21:46:26.816Z17/30 Write block 363 len 36864 data: 14 13 14 13 16 15 15 14 16
179272024-02-13T21:46:26.819Z18/30 Write block 542 len 40960 data: 17 16 12 14 13 11 9 13 14 19
179282024-02-13T21:46:26.819Z19/30 Flush
179292024-02-13T21:46:26.822Z20/30 Write block 17 len 36864 data: 14 15 13 14 15 16 14 14 14
179302024-02-13T21:46:26.822Z21/30 Read block 301 len 12288
179312024-02-13T21:46:26.825Z22/30 Read block 454 len 24576
179322024-02-13T21:46:26.825Z23/30 Read block 347 len 16384
179332024-02-13T21:46:26.825Z24/30 Read block 181 len 36864
179342024-02-13T21:46:26.828Z25/30 Write block 548 len 40960 data: 10 14 15 20 17 16 15 14 15 14
179352024-02-13T21:46:26.828Z26/30 Flush
179362024-02-13T21:46:26.831Z27/30 Write block 323 len 20480 data: 19 17 17 18 16
179372024-02-13T21:46:26.831Z28/30 Read block 435 len 40960
179382024-02-13T21:46:26.831Z29/30 Write block 63 len 12288 data: 15 10 10
179392024-02-13T21:46:26.834Z30/30 Write block 401 len 16384 data: 12 14 14 14
179402024-02-13T21:46:26.834Z----------------------------------------------------------------
179412024-02-13T21:46:26.834Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
179422024-02-13T21:46:26.835ZStates: Active Active Active
179432024-02-13T21:46:26.835ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179442024-02-13T21:46:26.835Z 1 Acked 1000 Write 10 Done Done Done false
179452024-02-13T21:46:26.835Z 2 Acked 1001 Read 5 Done Done Done false
179462024-02-13T21:46:26.835Z 3 Acked 1002 Write 9 Done Sent Done false
179472024-02-13T21:46:26.835Z 4 Acked 1003 Read 7 Done Sent Done false
179482024-02-13T21:46:26.835Z 5 Acked 1004 Read 4 Done Sent Done false
179492024-02-13T21:46:26.835Z 6 Acked 1005 Flush 0 Done Sent Done false
179502024-02-13T21:46:26.835Z 7 Acked 1006 Write 2 Done Sent Done false
179512024-02-13T21:46:26.835Z 8 Acked 1007 Read 1 Done Sent Done false
179522024-02-13T21:46:26.835Z 9 Acked 1008 Read 4 Done Sent Done false
179532024-02-13T21:46:26.835Z 10 Acked 1009 Write 2 Done Sent Done false
179542024-02-13T21:46:26.835Z 11 Acked 1010 Write 5 Done Sent Done false
179552024-02-13T21:46:26.835Z 12 Acked 1011 Write 2 Done Sent Done false
179562024-02-13T21:46:26.835Z 13 Acked 1012 Write 1 Done Sent Done false
179572024-02-13T21:46:26.835Z 14 Acked 1013 Write 5 Done Sent Done false
179582024-02-13T21:46:26.835Z 15 Acked 1014 Read 8 Done Sent Done false
179592024-02-13T21:46:26.835Z 16 Acked 1015 Read 9 Done Sent Done false
179602024-02-13T21:46:26.835Z 17 Acked 1016 Write 9 Done Sent Done false
179612024-02-13T21:46:26.835Z 18 Acked 1017 Write 10 Done Sent Done false
179622024-02-13T21:46:26.835Z 19 Acked 1018 Flush 0 Done Sent Done false
179632024-02-13T21:46:26.835Z 20 Acked 1019 Write 9 Done Sent Done false
179642024-02-13T21:46:26.835Z 21 Acked 1020 Read 3 Done Sent Done false
179652024-02-13T21:46:26.835Z 22 Acked 1021 Read 6 Done Sent Done false
179662024-02-13T21:46:26.836Z 23 Acked 1022 Read 4 Done Sent Done false
179672024-02-13T21:46:26.836Z 24 Acked 1023 Read 9 Done Sent Done false
179682024-02-13T21:46:26.836Z 25 Acked 1024 Write 10 Done Sent Done false
179692024-02-13T21:46:26.836Z 26 Acked 1025 Flush 0 Done Sent Done false
179702024-02-13T21:46:26.836Z 27 Acked 1026 Write 5 Done Sent Done false
179712024-02-13T21:46:26.836Z 28 Acked 1027 Read 10 Done Sent Done false
179722024-02-13T21:46:26.836Z 29 Acked 1028 Write 3 Done Sent Sent false
179732024-02-13T21:46:26.836Z 30 Acked 1029 Write 4 Sent Sent Sent false
179742024-02-13T21:46:26.836Z STATES DS:0 DS:1 DS:2 TOTAL
179752024-02-13T21:46:26.836Z New 0 0 0 0
179762024-02-13T21:46:26.836Z Sent 1 28 2 31
179772024-02-13T21:46:26.836Z Done 29 2 28 59
179782024-02-13T21:46:26.836Z Skipped 0 0 0 0
179792024-02-13T21:46:26.836Z Error 0 0 0 0
179802024-02-13T21:46:26.836ZLast Flush: 1025 0 1025
179812024-02-13T21:46:26.836ZDownstairs last five completed:
179822024-02-13T21:46:26.836ZUpstairs last five completed: 30 29 28 27 26
179832024-02-13T21:46:26.836Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:25.525683473Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1802}
179842024-02-13T21:46:26.836ZWrote out file "/tmp/test_repair_verify.data"
179852024-02-13T21:46:26.840Z
179862024-02-13T21:46:26.840ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179872024-02-13T21:46:26.855ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179882024-02-13T21:46:26.855Z 0 000-019 166 160 166 308 299 308 F F F <---
179892024-02-13T21:46:26.855Z 1 020-039 166 165 166 308 305 308 F F F <---
179902024-02-13T21:46:26.855Z 2 040-059 166 163 166 307 303 307 F F F <---
179912024-02-13T21:46:26.855Z 3 060-079 166 163 166 307 303 307 T T T <---
179922024-02-13T21:46:26.855Z 4 080-099 165 165 165 305 305 305 F F F
179932024-02-13T21:46:26.858Z 5 100-119 163 163 163 303 303 303 F F F
179942024-02-13T21:46:26.858Z 6 120-139 151 151 151 284 284 284 F F F
179952024-02-13T21:46:26.858Z 7 140-159 163 163 163 303 303 303 F F F
179962024-02-13T21:46:26.858Z 8 160-179 166 163 166 307 303 307 F F F <---
179972024-02-13T21:46:26.858Z 9 180-199 165 165 165 305 305 305 F F F
179982024-02-13T21:46:26.858Z 10 200-219 166 148 166 307 280 307 F F F <---
179992024-02-13T21:46:26.858Z 11 220-239 159 159 159 298 298 298 F F F
180002024-02-13T21:46:26.858Z 12 240-259 163 163 163 303 303 303 F F F
180012024-02-13T21:46:26.858Z 13 260-279 163 163 163 303 303 303 F F F
180022024-02-13T21:46:26.858Z 14 280-299 159 159 159 298 298 298 F F F
180032024-02-13T21:46:26.859Z 15 300-319 158 158 158 297 297 297 F F F
180042024-02-13T21:46:26.859Z 16 320-339 165 165 165 305 305 305 T F T <---
180052024-02-13T21:46:26.859Z 17 340-359 152 152 152 286 286 286 F F F
180062024-02-13T21:46:26.859Z 18 360-379 166 164 166 307 304 307 F F F <---
180072024-02-13T21:46:26.859Z 19 380-399 166 162 166 307 302 307 F F F <---
180082024-02-13T21:46:26.859Z 20 400-419 160 160 160 299 299 299 T F T <---
180092024-02-13T21:46:26.859Z 21 420-439 163 163 163 303 303 303 F F F
180102024-02-13T21:46:26.859Z 22 440-459 166 156 166 307 294 307 F F F <---
180112024-02-13T21:46:26.859Z 23 460-479 160 160 160 299 299 299 F F F
180122024-02-13T21:46:26.859Z 24 480-499 165 165 165 305 305 305 F F F
180132024-02-13T21:46:26.859Z 25 500-519 165 165 165 305 305 305 F F F
180142024-02-13T21:46:26.859Z 26 520-539 161 161 161 301 301 301 F F F
180152024-02-13T21:46:26.859Z 27 540-559 166 162 166 308 302 308 F F F <---
180162024-02-13T21:46:26.859Z 28 560-579 157 157 157 296 296 296 F F F
180172024-02-13T21:46:26.859Z 29 580-599 154 154 154 290 290 290 F F F
180182024-02-13T21:46:26.859ZMax gen: 166, Max flush: 308
180192024-02-13T21:46:26.859ZError: Difference in extent metadata found!
180202024-02-13T21:46:26.859ZA Difference in extent metadata is expected here
180212024-02-13T21:46:26.859ZOn loop 83
180222024-02-13T21:46:26.859Z
180232024-02-13T21:46:26.862ZVerifying data now
180242024-02-13T21:46:31.430ZNo change after verify
180252024-02-13T21:46:31.430ZLoop: 83 Downstairs dump after verify (and repair):
180262024-02-13T21:46:31.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180272024-02-13T21:46:31.445Z 0 000-019 166 166 166 308 308 308 F F F
180282024-02-13T21:46:31.445Z 1 020-039 166 166 166 308 308 308 F F F
180292024-02-13T21:46:31.445Z 2 040-059 166 166 166 307 307 307 F F F
180302024-02-13T21:46:31.445Z 3 060-079 167 167 167 309 309 309 F F F
180312024-02-13T21:46:31.445Z 4 080-099 165 165 165 305 305 305 F F F
180322024-02-13T21:46:31.445Z 5 100-119 163 163 163 303 303 303 F F F
180332024-02-13T21:46:31.445Z 6 120-139 151 151 151 284 284 284 F F F
180342024-02-13T21:46:31.445Z 7 140-159 163 163 163 303 303 303 F F F
180352024-02-13T21:46:31.445Z 8 160-179 166 166 166 307 307 307 F F F
180362024-02-13T21:46:31.448Z 9 180-199 165 165 165 305 305 305 F F F
180372024-02-13T21:46:31.448Z 10 200-219 166 166 166 307 307 307 F F F
180382024-02-13T21:46:31.448Z 11 220-239 159 159 159 298 298 298 F F F
180392024-02-13T21:46:31.448Z 12 240-259 163 163 163 303 303 303 F F F
180402024-02-13T21:46:31.449Z 13 260-279 163 163 163 303 303 303 F F F
180412024-02-13T21:46:31.449Z 14 280-299 159 159 159 298 298 298 F F F
180422024-02-13T21:46:31.449Z 15 300-319 158 158 158 297 297 297 F F F
180432024-02-13T21:46:31.449Z 16 320-339 167 167 167 309 309 309 F F F
180442024-02-13T21:46:31.449Z 17 340-359 152 152 152 286 286 286 F F F
180452024-02-13T21:46:31.449Z 18 360-379 166 166 166 307 307 307 F F F
180462024-02-13T21:46:31.449Z 19 380-399 166 166 166 307 307 307 F F F
180472024-02-13T21:46:31.449Z 20 400-419 167 167 167 309 309 309 F F F
180482024-02-13T21:46:31.449Z 21 420-439 163 163 163 303 303 303 F F F
180492024-02-13T21:46:31.449Z 22 440-459 166 166 166 307 307 307 F F F
180502024-02-13T21:46:31.449Z 23 460-479 160 160 160 299 299 299 F F F
180512024-02-13T21:46:31.449Z 24 480-499 165 165 165 305 305 305 F F F
180522024-02-13T21:46:31.449Z 25 500-519 165 165 165 305 305 305 F F F
180532024-02-13T21:46:31.449Z 26 520-539 161 161 161 301 301 301 F F F
180542024-02-13T21:46:31.449Z 27 540-559 166 166 166 308 308 308 F F F
180552024-02-13T21:46:31.449Z 28 560-579 157 157 157 296 296 296 F F F
180562024-02-13T21:46:31.449Z 29 580-599 154 154 154 290 290 290 F F F
180572024-02-13T21:46:31.449ZMax gen: 167, Max flush: 309
180582024-02-13T21:46:31.449Z
180592024-02-13T21:46:31.449ZBegin loop 84
180602024-02-13T21:46:31.449ZDownstairs to restart: 2
180612024-02-13T21:46:31.466Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157339854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
180622024-02-13T21:46:31.466Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157591273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
180632024-02-13T21:46:31.466Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.15762644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
180642024-02-13T21:46:31.470Z{"msg":"Crucible 2eecc974-092c-4e8c-9e00-1d4feb5202b4 has session id: 44838bba-687b-438f-944c-c2ef9eeca6a9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157651681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180652024-02-13T21:46:31.470Z{"msg":"Crucible runtime is spawned
180662024-02-13T21:46:31.470ZUpstairs opts: Upstairs UUID: 2eecc974-092c-4e8c-9e00-1d4feb5202b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157722995Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180672024-02-13T21:46:31.470Z{"msg":"Crucible stats registered with UUID: 2eecc974-092c-4e8c-9e00-1d4feb5202b4","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157760671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180682024-02-13T21:46:31.470Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157781784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
180692024-02-13T21:46:31.471Z{"msg":"2eecc974-092c-4e8c-9e00-1d4feb5202b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.15784711Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180702024-02-13T21:46:31.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157872631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180712024-02-13T21:46:31.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157888585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180722024-02-13T21:46:31.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157918354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180732024-02-13T21:46:31.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.157951642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180742024-02-13T21:46:31.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158029954Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180752024-02-13T21:46:31.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158058183Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180762024-02-13T21:46:31.471Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158085883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180772024-02-13T21:46:31.471Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158119181Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180782024-02-13T21:46:31.471Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158152349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180792024-02-13T21:46:31.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158183028Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180802024-02-13T21:46:31.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158214556Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180812024-02-13T21:46:31.471Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158250743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"io task","client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180822024-02-13T21:46:31.471Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158915572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180832024-02-13T21:46:31.471Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158951059Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180842024-02-13T21:46:31.471Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.158967553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180852024-02-13T21:46:31.471Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.15900291Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180862024-02-13T21:46:31.471Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.1595016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180872024-02-13T21:46:31.471Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.159614759Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180882024-02-13T21:46:31.471Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.159967681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180892024-02-13T21:46:31.471Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160016413Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180902024-02-13T21:46:31.471Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160057318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180912024-02-13T21:46:31.471Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160087007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180922024-02-13T21:46:31.471Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160102682Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180932024-02-13T21:46:31.472Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160130012Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180942024-02-13T21:46:31.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.160143697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180952024-02-13T21:46:31.472Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.16016388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180962024-02-13T21:46:31.472Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.16060324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180972024-02-13T21:46:31.476Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161040111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180982024-02-13T21:46:31.476Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161075079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
180992024-02-13T21:46:31.476Z{"msg":"[0]R flush_numbers[0..12]: [308, 308, 307, 309, 305, 303, 284, 303, 307, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161090893Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181002024-02-13T21:46:31.476Z{"msg":"[0]R generation[0..12]: [166, 166, 166, 167, 165, 163, 151, 163, 166, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161112905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181012024-02-13T21:46:31.476Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.16112768Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181022024-02-13T21:46:31.476Z{"msg":"[1]R flush_numbers[0..12]: [308, 308, 307, 309, 305, 303, 284, 303, 307, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161148852Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181032024-02-13T21:46:31.476Z{"msg":"[1]R generation[0..12]: [166, 166, 166, 167, 165, 163, 151, 163, 166, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161163007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181042024-02-13T21:46:31.476Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161177302Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181052024-02-13T21:46:31.476Z{"msg":"[2]R flush_numbers[0..12]: [308, 308, 307, 309, 305, 303, 284, 303, 307, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161192156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181062024-02-13T21:46:31.476Z{"msg":"[2]R generation[0..12]: [166, 166, 166, 167, 165, 163, 151, 163, 166, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161212419Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"Wait for a query_work_queue command to finish before sending IO
181072024-02-13T21:46:31.476Zdownstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181082024-02-13T21:46:31.476Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161229773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181092024-02-13T21:46:31.476Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161250165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181102024-02-13T21:46:31.476Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.16126401Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181112024-02-13T21:46:31.476Z{"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161277036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181122024-02-13T21:46:31.476Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161296918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"":"downstairs","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181132024-02-13T21:46:31.476Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161310533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181142024-02-13T21:46:31.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161329017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"0","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181152024-02-13T21:46:31.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161342502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"1","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181162024-02-13T21:46:31.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161369532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"client":"2","":"downstairs","session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181172024-02-13T21:46:31.476Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161383817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181182024-02-13T21:46:31.477Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.16140354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181192024-02-13T21:46:31.477Z{"msg":"2eecc974-092c-4e8c-9e00-1d4feb5202b4 is now active with session: 44838bba-687b-438f-944c-c2ef9eeca6a9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161416595Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181202024-02-13T21:46:31.477Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161435428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810,"session_id":"44838bba-687b-438f-944c-c2ef9eeca6a9"}
181212024-02-13T21:46:31.477Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:30.161447894Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
181222024-02-13T21:46:31.477ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181232024-02-13T21:46:31.477ZLoading write count information from file "/tmp/test_repair_verify.data"
181242024-02-13T21:46:31.477ZRun Repair workload
181252024-02-13T21:46:31.477Z01/30 Write block 291 len 32768 data: 12 14 15 14 12 10 12 12
181262024-02-13T21:46:31.477Z02/30 Write block 349 len 28672 data: 11 11 11 13 14 15 16
181272024-02-13T21:46:31.477Z03/30 Read block 186 len 4096
181282024-02-13T21:46:31.477Z04/30 Read block 328 len 4096
181292024-02-13T21:46:31.477Z05/30 Read block 371 len 32768
181302024-02-13T21:46:31.480Z06/30 Write block 542 len 4096 data: 18
181312024-02-13T21:46:31.480Z07/30 Read block 106 len 12288
181322024-02-13T21:46:31.480Z08/30 Write block 522 len 16384 data: 17 16 12 10
181332024-02-13T21:46:31.480Z09/30 Write block 161 len 8192 data: 13 13
181342024-02-13T21:46:31.480Z10/30 Read block 317 len 40960
181352024-02-13T21:46:31.481Z11/30 Write block 82 len 36864 data: 12 10 10 11 11 11 12 12 11
181362024-02-13T21:46:31.481Z12/30 Write block 507 len 16384 data: 13 13 14 15
181372024-02-13T21:46:31.481Z13/30 Read block 506 len 32768
181382024-02-13T21:46:31.484Z14/30 Read block 118 len 20480
181392024-02-13T21:46:31.484Z15/30 Write block 152 len 32768 data: 9 12 12 13 11 11 12 11
181402024-02-13T21:46:31.484Z16/30 Read block 369 len 32768
181412024-02-13T21:46:31.487Z17/30 Read block 595 len 16384
181422024-02-13T21:46:31.487Z18/30 Read block 251 len 4096
181432024-02-13T21:46:31.487Z19/30 Read block 367 len 40960
181442024-02-13T21:46:31.487Z20/30 Write block 433 len 8192 data: 14 15
181452024-02-13T21:46:31.490Z21/30 Read block 475 len 24576
181462024-02-13T21:46:31.490Z22/30 Read block 54 len 32768
181472024-02-13T21:46:31.490Z23/30 Read block 254 len 8192
181482024-02-13T21:46:31.490Z24/30 Read block 506 len 32768
181492024-02-13T21:46:31.496Z25/30 Read block 242 len 36864
181502024-02-13T21:46:31.496Z26/30 Read block 238 len 12288
181512024-02-13T21:46:31.496Z27/30 Write block 44 len 20480 data: 8 8 10 9 10
181522024-02-13T21:46:31.496Z28/30 Read block 306 len 32768
181532024-02-13T21:46:31.499Z29/30 Write block 12 len 8192 data: 14 14
181542024-02-13T21:46:31.499Z30/30 Read block 41 len 12288
181552024-02-13T21:46:31.499Z----------------------------------------------------------------
181562024-02-13T21:46:31.499Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
181572024-02-13T21:46:31.499ZStates: Active Active Active
181582024-02-13T21:46:31.500ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181592024-02-13T21:46:31.500Z 1 Acked 1000 Write 8 Done Done Sent false
181602024-02-13T21:46:31.500Z 2 Acked 1001 Write 7 Done Done Sent false
181612024-02-13T21:46:31.500Z 3 Acked 1002 Read 1 Done Done Sent false
181622024-02-13T21:46:31.500Z 4 Acked 1003 Read 1 Done Done Sent false
181632024-02-13T21:46:31.500Z 5 Acked 1004 Read 8 Done Done Sent false
181642024-02-13T21:46:31.500Z 6 Acked 1005 Write 1 Done Done Sent false
181652024-02-13T21:46:31.500Z 7 Acked 1006 Read 3 Done Done Sent false
181662024-02-13T21:46:31.500Z 8 Acked 1007 Write 4 Done Done Sent false
181672024-02-13T21:46:31.500Z 9 Acked 1008 Write 2 Done Done Sent false
181682024-02-13T21:46:31.500Z 10 Acked 1009 Read 10 Done Done Sent false
181692024-02-13T21:46:31.500Z 11 Acked 1010 Write 9 Done Done Sent false
181702024-02-13T21:46:31.500Z 12 Acked 1011 Write 4 Done Done Sent false
181712024-02-13T21:46:31.500Z 13 Acked 1012 Read 8 Done Done Sent false
181722024-02-13T21:46:31.500Z 14 Acked 1013 Read 5 Done Done Sent false
181732024-02-13T21:46:31.500Z 15 Acked 1014 Write 8 Done Done Sent false
181742024-02-13T21:46:31.500Z 16 Acked 1015 Read 8 Done Done Sent false
181752024-02-13T21:46:31.500Z 17 Acked 1016 Read 4 Done Done Sent false
181762024-02-13T21:46:31.500Z 18 Acked 1017 Read 1 Done Done Sent false
181772024-02-13T21:46:31.500Z 19 Acked 1018 Read 10 Done Done Sent false
181782024-02-13T21:46:31.500Z 20 Acked 1019 Write 2 Done Done Sent false
181792024-02-13T21:46:31.500Z 21 Acked 1020 Read 6 Done Done Sent false
181802024-02-13T21:46:31.500Z 22 Acked 1021 Read 8 Done Done Sent false
181812024-02-13T21:46:31.500Z 23 Acked 1022 Read 2 Done Done Sent false
181822024-02-13T21:46:31.501Z 24 Acked 1023 Read 8 Done Done Sent false
181832024-02-13T21:46:31.501Z 25 Acked 1024 Read 9 Done Done Sent false
181842024-02-13T21:46:31.501Z 26 Acked 1025 Read 3 Done Done Sent false
181852024-02-13T21:46:31.501Z 27 Acked 1026 Write 5 Done Done Sent false
181862024-02-13T21:46:31.501Z 28 Acked 1027 Read 8 Done Done Sent false
181872024-02-13T21:46:31.501Z 29 Acked 1028 Write 2 Done Done Sent false
181882024-02-13T21:46:31.501Z 30 Acked 1029 Read 3 Sent Done Sent false
181892024-02-13T21:46:31.501Z STATES DS:0 DS:1 DS:2 TOTAL
181902024-02-13T21:46:31.501Z New 0 0 0 0
181912024-02-13T21:46:31.501Z Sent 1 0 30 31
181922024-02-13T21:46:31.501Z Done 29 30 0 59
181932024-02-13T21:46:31.501Z Skipped 0 0 0 0
181942024-02-13T21:46:31.501Z Error 0 0 0 0
181952024-02-13T21:46:31.501ZLast Flush: 0 0 0
181962024-02-13T21:46:31.501ZDownstairs last five completed:
181972024-02-13T21:46:31.501ZUpstairs last five completed: 30 29 28 27 26
181982024-02-13T21:46:31.501Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:30.187473903Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1810}
181992024-02-13T21:46:31.501ZWrote out file "/tmp/test_repair_verify.data"
182002024-02-13T21:46:31.504Z
182012024-02-13T21:46:31.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182022024-02-13T21:46:31.521ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182032024-02-13T21:46:31.521Z 0 000-019 166 166 166 308 308 308 T T F <---
182042024-02-13T21:46:31.521Z 1 020-039 166 166 166 308 308 308 F F F
182052024-02-13T21:46:31.521Z 2 040-059 166 166 166 307 307 307 T T F <---
182062024-02-13T21:46:31.521Z 3 060-079 167 167 167 309 309 309 F F F
182072024-02-13T21:46:31.521Z 4 080-099 165 165 165 305 305 305 T T F <---
182082024-02-13T21:46:31.521Z 5 100-119 163 163 163 303 303 303 F F F
182092024-02-13T21:46:31.521Z 6 120-139 151 151 151 284 284 284 F F F
182102024-02-13T21:46:31.521Z 7 140-159 163 163 163 303 303 303 T T F <---
182112024-02-13T21:46:31.521Z 8 160-179 166 166 166 307 307 307 T T F <---
182122024-02-13T21:46:31.521Z 9 180-199 165 165 165 305 305 305 F F F
182132024-02-13T21:46:31.524Z 10 200-219 166 166 166 307 307 307 F F F
182142024-02-13T21:46:31.524Z 11 220-239 159 159 159 298 298 298 F F F
182152024-02-13T21:46:31.524Z 12 240-259 163 163 163 303 303 303 F F F
182162024-02-13T21:46:31.524Z 13 260-279 163 163 163 303 303 303 F F F
182172024-02-13T21:46:31.525Z 14 280-299 159 159 159 298 298 298 T T F <---
182182024-02-13T21:46:31.525Z 15 300-319 158 158 158 297 297 297 F F F
182192024-02-13T21:46:31.525Z 16 320-339 167 167 167 309 309 309 F F F
182202024-02-13T21:46:31.525Z 17 340-359 152 152 152 286 286 286 T T F <---
182212024-02-13T21:46:31.525Z 18 360-379 166 166 166 307 307 307 F F F
182222024-02-13T21:46:31.525Z 19 380-399 166 166 166 307 307 307 F F F
182232024-02-13T21:46:31.525Z 20 400-419 167 167 167 309 309 309 F F F
182242024-02-13T21:46:31.525Z 21 420-439 163 163 163 303 303 303 T T F <---
182252024-02-13T21:46:31.525Z 22 440-459 166 166 166 307 307 307 F F F
182262024-02-13T21:46:31.525Z 23 460-479 160 160 160 299 299 299 F F F
182272024-02-13T21:46:31.525Z 24 480-499 165 165 165 305 305 305 F F F
182282024-02-13T21:46:31.525Z 25 500-519 165 165 165 305 305 305 T T F <---
182292024-02-13T21:46:31.525Z 26 520-539 161 161 161 301 301 301 T T F <---
182302024-02-13T21:46:31.525Z 27 540-559 166 166 166 308 308 308 T T F <---
182312024-02-13T21:46:31.525Z 28 560-579 157 157 157 296 296 296 F F F
182322024-02-13T21:46:31.525Z 29 580-599 154 154 154 290 290 290 F F F
182332024-02-13T21:46:31.525ZMax gen: 167, Max flush: 309
182342024-02-13T21:46:31.525ZError: Difference in extent metadata found!
182352024-02-13T21:46:31.525ZA Difference in extent metadata is expected here
182362024-02-13T21:46:31.525ZOn loop 84
182372024-02-13T21:46:31.525Z
182382024-02-13T21:46:31.529ZVerifying data now
182392024-02-13T21:46:36.092ZNo change after verify
182402024-02-13T21:46:36.092ZLoop: 84 Downstairs dump after verify (and repair):
182412024-02-13T21:46:36.107ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182422024-02-13T21:46:36.107Z 0 000-019 168 168 168 310 310 310 F F F
182432024-02-13T21:46:36.107Z 1 020-039 166 166 166 308 308 308 F F F
182442024-02-13T21:46:36.107Z 2 040-059 168 168 168 310 310 310 F F F
182452024-02-13T21:46:36.107Z 3 060-079 167 167 167 309 309 309 F F F
182462024-02-13T21:46:36.107Z 4 080-099 168 168 168 310 310 310 F F F
182472024-02-13T21:46:36.107Z 5 100-119 163 163 163 303 303 303 F F F
182482024-02-13T21:46:36.107Z 6 120-139 151 151 151 284 284 284 F F F
182492024-02-13T21:46:36.107Z 7 140-159 168 168 168 310 310 310 F F F
182502024-02-13T21:46:36.107Z 8 160-179 168 168 168 310 310 310 F F F
182512024-02-13T21:46:36.111Z 9 180-199 165 165 165 305 305 305 F F F
182522024-02-13T21:46:36.111Z 10 200-219 166 166 166 307 307 307 F F F
182532024-02-13T21:46:36.111Z 11 220-239 159 159 159 298 298 298 F F F
182542024-02-13T21:46:36.111Z 12 240-259 163 163 163 303 303 303 F F F
182552024-02-13T21:46:36.111Z 13 260-279 163 163 163 303 303 303 F F F
182562024-02-13T21:46:36.111Z 14 280-299 168 168 168 310 310 310 F F F
182572024-02-13T21:46:36.111Z 15 300-319 158 158 158 297 297 297 F F F
182582024-02-13T21:46:36.111Z 16 320-339 167 167 167 309 309 309 F F F
182592024-02-13T21:46:36.111Z 17 340-359 168 168 168 310 310 310 F F F
182602024-02-13T21:46:36.111Z 18 360-379 166 166 166 307 307 307 F F F
182612024-02-13T21:46:36.111Z 19 380-399 166 166 166 307 307 307 F F F
182622024-02-13T21:46:36.111Z 20 400-419 167 167 167 309 309 309 F F F
182632024-02-13T21:46:36.111Z 21 420-439 168 168 168 310 310 310 F F F
182642024-02-13T21:46:36.111Z 22 440-459 166 166 166 307 307 307 F F F
182652024-02-13T21:46:36.111Z 23 460-479 160 160 160 299 299 299 F F F
182662024-02-13T21:46:36.111Z 24 480-499 165 165 165 305 305 305 F F F
182672024-02-13T21:46:36.111Z 25 500-519 168 168 168 310 310 310 F F F
182682024-02-13T21:46:36.111Z 26 520-539 168 168 168 310 310 310 F F F
182692024-02-13T21:46:36.112Z 27 540-559 168 168 168 310 310 310 F F F
182702024-02-13T21:46:36.112Z 28 560-579 157 157 157 296 296 296 F F F
182712024-02-13T21:46:36.112Z 29 580-599 154 154 154 290 290 290 F F F
182722024-02-13T21:46:36.112ZMax gen: 168, Max flush: 310
182732024-02-13T21:46:36.112Z
182742024-02-13T21:46:36.112ZBegin loop 85
182752024-02-13T21:46:36.112ZDownstairs to restart: 1
182762024-02-13T21:46:36.128Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818140793Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
182772024-02-13T21:46:36.128Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818418232Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
182782024-02-13T21:46:36.128Z{Crucible runtime is spawned
182792024-02-13T21:46:36.133Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818489637Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
182802024-02-13T21:46:36.133Z{"msg":"Crucible 81b7a8fa-d9f0-48a3-957c-6e26a6ffbe2b has session id: 33abab7a-558e-4382-bf81-f4c466585d63","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.81853498Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182812024-02-13T21:46:36.133Z{"msg":"Upstairs opts: Upstairs UUID: 81b7a8fa-d9f0-48a3-957c-6e26a6ffbe2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818583863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182822024-02-13T21:46:36.133Z{"msg":"Crucible stats registered with UUID: 81b7a8fa-d9f0-48a3-957c-6e26a6ffbe2b","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818603615Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182832024-02-13T21:46:36.133Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818628706Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
182842024-02-13T21:46:36.133Z{"msg":"81b7a8fa-d9f0-48a3-957c-6e26a6ffbe2b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818753641Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182852024-02-13T21:46:36.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818785319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182862024-02-13T21:46:36.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818804283Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182872024-02-13T21:46:36.133Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818821516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182882024-02-13T21:46:36.133Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.81883905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182892024-02-13T21:46:36.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818886893Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182902024-02-13T21:46:36.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818929077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182912024-02-13T21:46:36.134Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.818972682Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182922024-02-13T21:46:36.134Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819019345Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182932024-02-13T21:46:36.134Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819054692Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182942024-02-13T21:46:36.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819081032Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182952024-02-13T21:46:36.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819105603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182962024-02-13T21:46:36.134Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819219172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"io task","client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182972024-02-13T21:46:36.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819765714Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182982024-02-13T21:46:36.134Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819822753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
182992024-02-13T21:46:36.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819855351Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183002024-02-13T21:46:36.134Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.819882881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183012024-02-13T21:46:36.134Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820342255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183022024-02-13T21:46:36.134Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.82049383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183032024-02-13T21:46:36.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820625912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183042024-02-13T21:46:36.134Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820672295Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183052024-02-13T21:46:36.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820750427Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183062024-02-13T21:46:36.134Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820813014Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183072024-02-13T21:46:36.134Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820843203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183082024-02-13T21:46:36.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820951124Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183092024-02-13T21:46:36.134Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820982522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183102024-02-13T21:46:36.134Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.820997327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183112024-02-13T21:46:36.134Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821200453Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183122024-02-13T21:46:36.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821604897Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183132024-02-13T21:46:36.134Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821643902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183142024-02-13T21:46:36.135Z{"msg":"[0]R flush_numbers[0..12]: [310, 308, 310, 309, 310, 303, 284, 310, 310, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821662166Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183152024-02-13T21:46:36.135Z{"msg":"[0]R generation[0..12]: [168, 166, 168, 167, 168, 163, 151, 168, 168, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821676781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183162024-02-13T21:46:36.135Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821699002Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183172024-02-13T21:46:36.135Z{"msg":"[1]R flush_numbers[0..12]: [310, 308, 310, 309, 310, 303, 284, 310, 310, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821723534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183182024-02-13T21:46:36.135Z{"msg":"[1]R generation[0..12]: [168, 166, 168, 167, 168, 163, 151, 168, 168, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821737848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183192024-02-13T21:46:36.135Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821759491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183202024-02-13T21:46:36.135Z{"msg":"[2]R flush_numbers[0..12]: [310, 308, 310, 309, 310, 303, 284, 310, 310, 305, 307, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821773955Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183212024-02-13T21:46:36.135Z{"msg":"[2]R generation[0..12]: [168, 166, 168, 167, 168, 163, 151, 168, 168, 165, 166, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.82178785Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183222024-02-13T21:46:36.135Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821808683Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183232024-02-13T21:46:36.135Z{"msg":"Max found gen is 169","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821822578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}Wait for a query_work_queue command to finish before sending IO
183242024-02-13T21:46:36.135Z
183252024-02-13T21:46:36.139Z{"msg":"Generation requested: 170 >= found:169","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821848758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183262024-02-13T21:46:36.139Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821876468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183272024-02-13T21:46:36.139Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821890833Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"":"downstairs","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183282024-02-13T21:46:36.139Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821904408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183292024-02-13T21:46:36.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821929069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"0","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183302024-02-13T21:46:36.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821949092Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"1","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183312024-02-13T21:46:36.139Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821965706Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"client":"2","":"downstairs","session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183322024-02-13T21:46:36.139Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.821989707Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183332024-02-13T21:46:36.139Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.822002952Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183342024-02-13T21:46:36.139Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.822015568Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
183352024-02-13T21:46:36.139Z{"msg":"81b7a8fa-d9f0-48a3-957c-6e26a6ffbe2b is now active with session: 33abab7a-558e-4382-bf81-f4c466585d63","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.822033931Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183362024-02-13T21:46:36.139Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:34.822052304Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818,"session_id":"33abab7a-558e-4382-bf81-f4c466585d63"}
183372024-02-13T21:46:36.139ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183382024-02-13T21:46:36.140ZLoading write count information from file "/tmp/test_repair_verify.data"
183392024-02-13T21:46:36.140ZRun Repair workload
183402024-02-13T21:46:36.140Z01/30 Write block 540 len 20480 data: 22 19 19 17 13
183412024-02-13T21:46:36.140Z02/30 Read block 160 len 36864
183422024-02-13T21:46:36.140Z03/30 Read block 201 len 4096
183432024-02-13T21:46:36.140Z04/30 Write block 493 len 36864 data: 12 14 12 13 14 13 11 12 8
183442024-02-13T21:46:36.140Z05/30 Write block 516 len 16384 data: 17 20 20 22
183452024-02-13T21:46:36.140Z06/30 Write block 333 len 24576 data: 16 12 16 15 12 11
183462024-02-13T21:46:36.140Z07/30 Write block 214 len 20480 data: 12 15 18 20 20
183472024-02-13T21:46:36.143Z08/30 Write block 481 len 40960 data: 12 14 13 12 11 12 11 12 12 10
183482024-02-13T21:46:36.143Z09/30 Write block 227 len 36864 data: 14 13 11 11 11 9 8 11 11
183492024-02-13T21:46:36.143Z10/30 Read block 119 len 36864
183502024-02-13T21:46:36.143Z11/30 Write block 205 len 4096 data: 9
183512024-02-13T21:46:36.143Z12/30 Read block 597 len 12288
183522024-02-13T21:46:36.147Z13/30 Write block 5 len 24576 data: 10 10 10 9 11 12
183532024-02-13T21:46:36.147Z14/30 Read block 393 len 24576
183542024-02-13T21:46:36.147Z15/30 Read block 35 len 8192
183552024-02-13T21:46:36.147Z16/30 Flush
183562024-02-13T21:46:36.150Z17/30 Write block 452 len 40960 data: 12 14 14 14 17 17 17 17 19 17
183572024-02-13T21:46:36.150Z18/30 Flush
183582024-02-13T21:46:36.153Z19/30 Write block 43 len 4096 data: 10
183592024-02-13T21:46:36.153Z20/30 Read block 149 len 8192
183602024-02-13T21:46:36.153Z21/30 Read block 164 len 24576
183612024-02-13T21:46:36.156Z22/30 Write block 207 len 4096 data: 6
183622024-02-13T21:46:36.156Z23/30 Read block 235 len 16384
183632024-02-13T21:46:36.156Z24/30 Read block 457 len 32768
183642024-02-13T21:46:36.156Z25/30 Write block 50 len 40960 data: 11 11 11 11 12 14 15 15 16 19
183652024-02-13T21:46:36.159Z26/30 Write block 219 len 40960 data: 19 19 17 19 20 18 16 13 15 14
183662024-02-13T21:46:36.159Z27/30 Read block 160 len 20480
183672024-02-13T21:46:36.159Z28/30 Read block 167 len 32768
183682024-02-13T21:46:36.163Z29/30 Write block 576 len 16384 data: 7 8 7 5
183692024-02-13T21:46:36.163Z30/30 Read block 334 len 12288
183702024-02-13T21:46:36.163Z----------------------------------------------------------------
183712024-02-13T21:46:36.163Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183722024-02-13T21:46:36.163ZStates: Active Active Active
183732024-02-13T21:46:36.163ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183742024-02-13T21:46:36.163Z 1 Acked 1000 Write 5 Done Sent Done false
183752024-02-13T21:46:36.163Z 2 Acked 1001 Read 9 Done Sent Done false
183762024-02-13T21:46:36.163Z 3 Acked 1002 Read 1 Done Sent Done false
183772024-02-13T21:46:36.163Z 4 Acked 1003 Write 9 Done Sent Done false
183782024-02-13T21:46:36.163Z 5 Acked 1004 Write 4 Done Sent Done false
183792024-02-13T21:46:36.163Z 6 Acked 1005 Write 6 Done Sent Done false
183802024-02-13T21:46:36.163Z 7 Acked 1006 Write 5 Done Sent Done false
183812024-02-13T21:46:36.163Z 8 Acked 1007 Write 10 Done Sent Done false
183822024-02-13T21:46:36.163Z 9 Acked 1008 Write 9 Done Sent Done false
183832024-02-13T21:46:36.163Z 10 Acked 1009 Read 9 Done Sent Done false
183842024-02-13T21:46:36.163Z 11 Acked 1010 Write 1 Done Sent Done false
183852024-02-13T21:46:36.163Z 12 Acked 1011 Read 3 Done Sent Done false
183862024-02-13T21:46:36.163Z 13 Acked 1012 Write 6 Done Sent Done false
183872024-02-13T21:46:36.163Z 14 Acked 1013 Read 6 Done Sent Done false
183882024-02-13T21:46:36.164Z 15 Acked 1014 Read 2 Done Sent Done false
183892024-02-13T21:46:36.164Z 16 Acked 1015 Flush 0 Done Sent Done false
183902024-02-13T21:46:36.164Z 17 Acked 1016 Write 10 Done Sent Done false
183912024-02-13T21:46:36.164Z 18 Acked 1017 Flush 0 Done Sent Done false
183922024-02-13T21:46:36.164Z 19 Acked 1018 Write 1 Done Sent Done false
183932024-02-13T21:46:36.164Z 20 Acked 1019 Read 2 Done Sent Done false
183942024-02-13T21:46:36.164Z 21 Acked 1020 Read 6 Done Sent Done false
183952024-02-13T21:46:36.164Z 22 Acked 1021 Write 1 Done Sent Done false
183962024-02-13T21:46:36.164Z 23 Acked 1022 Read 4 Done Sent Done false
183972024-02-13T21:46:36.164Z 24 Acked 1023 Read 8 Done Sent Done false
183982024-02-13T21:46:36.164Z 25 Acked 1024 Write 10 Done Sent Done false
183992024-02-13T21:46:36.164Z 26 Acked 1025 Write 10 Done Sent Done false
184002024-02-13T21:46:36.164Z 27 Acked 1026 Read 5 Done Sent Done false
184012024-02-13T21:46:36.164Z 28 Acked 1027 Read 8 Done Sent Done false
184022024-02-13T21:46:36.164Z 29 Acked 1028 Write 4 Done Sent Done false
184032024-02-13T21:46:36.164Z 30 Acked 1029 Read 3 Sent Sent Done false
184042024-02-13T21:46:36.164Z STATES DS:0 DS:1 DS:2 TOTAL
184052024-02-13T21:46:36.164Z New 0 0 0 0
184062024-02-13T21:46:36.164Z Sent 1 30 0 31
184072024-02-13T21:46:36.164Z Done 29 0 30 59
184082024-02-13T21:46:36.164Z Skipped 0 0 0 0
184092024-02-13T21:46:36.164Z Error 0 0 0 0
184102024-02-13T21:46:36.164ZLast Flush: 1017 0 1017
184112024-02-13T21:46:36.164ZDownstairs last five completed:
184122024-02-13T21:46:36.164ZUpstairs last five completed: 30 29 28 27 26
184132024-02-13T21:46:36.165Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:34.851548805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1818}
184142024-02-13T21:46:36.165ZWrote out file "/tmp/test_repair_verify.data"
184152024-02-13T21:46:36.168Z
184162024-02-13T21:46:36.171ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184172024-02-13T21:46:36.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184182024-02-13T21:46:36.184Z 0 000-019 170 168 170 311 310 311 F F F <---
184192024-02-13T21:46:36.184Z 1 020-039 166 166 166 308 308 308 F F F
184202024-02-13T21:46:36.184Z 2 040-059 168 168 168 310 310 310 T F T <---
184212024-02-13T21:46:36.185Z 3 060-079 167 167 167 309 309 309 F F F
184222024-02-13T21:46:36.185Z 4 080-099 168 168 168 310 310 310 F F F
184232024-02-13T21:46:36.185Z 5 100-119 163 163 163 303 303 303 F F F
184242024-02-13T21:46:36.185Z 6 120-139 151 151 151 284 284 284 F F F
184252024-02-13T21:46:36.185Z 7 140-159 168 168 168 310 310 310 F F F
184262024-02-13T21:46:36.185Z 8 160-179 168 168 168 310 310 310 F F F
184272024-02-13T21:46:36.185Z 9 180-199 165 165 165 305 305 305 F F F
184282024-02-13T21:46:36.191Z 10 200-219 170 166 170 311 307 311 T F T <---
184292024-02-13T21:46:36.191Z 11 220-239 170 159 170 311 298 311 T F T <---
184302024-02-13T21:46:36.191Z 12 240-259 163 163 163 303 303 303 F F F
184312024-02-13T21:46:36.191Z 13 260-279 163 163 163 303 303 303 F F F
184322024-02-13T21:46:36.191Z 14 280-299 168 168 168 310 310 310 F F F
184332024-02-13T21:46:36.191Z 15 300-319 158 158 158 297 297 297 F F F
184342024-02-13T21:46:36.191Z 16 320-339 170 167 170 311 309 311 F F F <---
184352024-02-13T21:46:36.191Z 17 340-359 168 168 168 310 310 310 F F F
184362024-02-13T21:46:36.191Z 18 360-379 166 166 166 307 307 307 F F F
184372024-02-13T21:46:36.191Z 19 380-399 166 166 166 307 307 307 F F F
184382024-02-13T21:46:36.191Z 20 400-419 167 167 167 309 309 309 F F F
184392024-02-13T21:46:36.191Z 21 420-439 168 168 168 310 310 310 F F F
184402024-02-13T21:46:36.191Z 22 440-459 170 166 170 312 307 312 F F F <---
184412024-02-13T21:46:36.191Z 23 460-479 170 160 170 312 299 312 F F F <---
184422024-02-13T21:46:36.191Z 24 480-499 170 165 170 311 305 311 F F F <---
184432024-02-13T21:46:36.191Z 25 500-519 170 168 170 311 310 311 F F F <---
184442024-02-13T21:46:36.191Z 26 520-539 168 168 168 310 310 310 F F F
184452024-02-13T21:46:36.192Z 27 540-559 170 168 170 311 310 311 F F F <---
184462024-02-13T21:46:36.192Z 28 560-579 157 157 157 296 296 296 T F T <---
184472024-02-13T21:46:36.192Z 29 580-599 154 154 154 290 290 290 F F F
184482024-02-13T21:46:36.192ZMax gen: 170, Max flush: 312
184492024-02-13T21:46:36.192ZError: Difference in extent metadata found!
184502024-02-13T21:46:36.192ZA Difference in extent metadata is expected here
184512024-02-13T21:46:36.192ZOn loop 85
184522024-02-13T21:46:36.192Z
184532024-02-13T21:46:36.195ZVerifying data now
184542024-02-13T21:46:40.738ZNo change after verify
184552024-02-13T21:46:40.741ZLoop: 85 Downstairs dump after verify (and repair):
184562024-02-13T21:46:40.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184572024-02-13T21:46:40.753Z 0 000-019 170 170 170 311 311 311 F F F
184582024-02-13T21:46:40.753Z 1 020-039 166 166 166 308 308 308 F F F
184592024-02-13T21:46:40.753Z 2 040-059 171 171 171 313 313 313 F F F
184602024-02-13T21:46:40.753Z 3 060-079 167 167 167 309 309 309 F F F
184612024-02-13T21:46:40.753Z 4 080-099 168 168 168 310 310 310 F F F
184622024-02-13T21:46:40.753Z 5 100-119 163 163 163 303 303 303 F F F
184632024-02-13T21:46:40.753Z 6 120-139 151 151 151 284 284 284 F F F
184642024-02-13T21:46:40.753Z 7 140-159 168 168 168 310 310 310 F F F
184652024-02-13T21:46:40.753Z 8 160-179 168 168 168 310 310 310 F F F
184662024-02-13T21:46:40.757Z 9 180-199 165 165 165 305 305 305 F F F
184672024-02-13T21:46:40.757Z 10 200-219 171 171 171 313 313 313 F F F
184682024-02-13T21:46:40.757Z 11 220-239 171 171 171 313 313 313 F F F
184692024-02-13T21:46:40.757Z 12 240-259 163 163 163 303 303 303 F F F
184702024-02-13T21:46:40.757Z 13 260-279 163 163 163 303 303 303 F F F
184712024-02-13T21:46:40.757Z 14 280-299 168 168 168 310 310 310 F F F
184722024-02-13T21:46:40.757Z 15 300-319 158 158 158 297 297 297 F F F
184732024-02-13T21:46:40.757Z 16 320-339 170 170 170 311 311 311 F F F
184742024-02-13T21:46:40.757Z 17 340-359 168 168 168 310 310 310 F F F
184752024-02-13T21:46:40.757Z 18 360-379 166 166 166 307 307 307 F F F
184762024-02-13T21:46:40.757Z 19 380-399 166 166 166 307 307 307 F F F
184772024-02-13T21:46:40.757Z 20 400-419 167 167 167 309 309 309 F F F
184782024-02-13T21:46:40.757Z 21 420-439 168 168 168 310 310 310 F F F
184792024-02-13T21:46:40.757Z 22 440-459 170 170 170 312 312 312 F F F
184802024-02-13T21:46:40.757Z 23 460-479 170 170 170 312 312 312 F F F
184812024-02-13T21:46:40.757Z 24 480-499 170 170 170 311 311 311 F F F
184822024-02-13T21:46:40.757Z 25 500-519 170 170 170 311 311 311 F F F
184832024-02-13T21:46:40.757Z 26 520-539 168 168 168 310 310 310 F F F
184842024-02-13T21:46:40.757Z 27 540-559 170 170 170 311 311 311 F F F
184852024-02-13T21:46:40.757Z 28 560-579 171 171 171 313 313 313 F F F
184862024-02-13T21:46:40.758Z 29 580-599 154 154 154 290 290 290 F F F
184872024-02-13T21:46:40.758ZMax gen: 171, Max flush: 313
184882024-02-13T21:46:40.758Z
184892024-02-13T21:46:40.758ZBegin loop 86
184902024-02-13T21:46:40.758ZDownstairs to restart: 0
184912024-02-13T21:46:40.775Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463259432Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826Crucible runtime is spawned
184922024-02-13T21:46:40.775Z}
184932024-02-13T21:46:40.780Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463584134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826}
184942024-02-13T21:46:40.780Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463627258Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826}
184952024-02-13T21:46:40.780Z{"msg":"Crucible e58f4af2-7ad3-405f-a8c0-4d099c348dfa has session id: 0d019e4f-73ae-4621-b67e-1c086bf8fa97","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463653359Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
184962024-02-13T21:46:40.780Z{"msg":"Upstairs opts: Upstairs UUID: e58f4af2-7ad3-405f-a8c0-4d099c348dfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463703381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
184972024-02-13T21:46:40.780Z{"msg":"Crucible stats registered with UUID: e58f4af2-7ad3-405f-a8c0-4d099c348dfa","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.46373219Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
184982024-02-13T21:46:40.780Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463755572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826}
184992024-02-13T21:46:40.780Z{"msg":"e58f4af2-7ad3-405f-a8c0-4d099c348dfa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463786961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185002024-02-13T21:46:40.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463808523Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185012024-02-13T21:46:40.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463830914Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185022024-02-13T21:46:40.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463853496Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185032024-02-13T21:46:40.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463888144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185042024-02-13T21:46:40.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463921072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185052024-02-13T21:46:40.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463944153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185062024-02-13T21:46:40.780Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.463985518Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185072024-02-13T21:46:40.780Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464014118Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185082024-02-13T21:46:40.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464053324Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185092024-02-13T21:46:40.781Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:39.464081374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185102024-02-13T21:46:40.781Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:39.46411845Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185112024-02-13T21:46:40.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464260279Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185122024-02-13T21:46:40.781Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:39.464306932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185132024-02-13T21:46:40.781Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464329264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185142024-02-13T21:46:40.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464351096Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185152024-02-13T21:46:40.781Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464391501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185162024-02-13T21:46:40.781Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464415812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185172024-02-13T21:46:40.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464861491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185182024-02-13T21:46:40.781Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464907874Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185192024-02-13T21:46:40.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.464936444Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185202024-02-13T21:46:40.781Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.46497282Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185212024-02-13T21:46:40.781Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.465480936Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185222024-02-13T21:46:40.781Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.46552629Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185232024-02-13T21:46:40.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466030227Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185242024-02-13T21:46:40.781Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466067513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185252024-02-13T21:46:40.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466083997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185262024-02-13T21:46:40.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466152472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185272024-02-13T21:46:40.781Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466191268Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185282024-02-13T21:46:40.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:39.466207502Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185292024-02-13T21:46:41.777Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.464843971Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185302024-02-13T21:46:41.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.465035481Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"io task","client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185312024-02-13T21:46:41.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.465683116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185322024-02-13T21:46:41.781Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.465718813Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185332024-02-13T21:46:41.782Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466204277Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185342024-02-13T21:46:41.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.46663834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185352024-02-13T21:46:41.782Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466668079Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185362024-02-13T21:46:41.782Z{"msg":"[0]R flush_numbers[0..12]: [311, 308, 313, 309, 310, 303, 284, 310, 310, 305, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466683443Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185372024-02-13T21:46:41.782Z{"msg":"[0]R generation[0..12]: [170, 166, 171, 167, 168, 163, 151, 168, 168, 165, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466697978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185382024-02-13T21:46:41.782Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466723669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185392024-02-13T21:46:41.782Z{"msg":"[1]R flush_numbers[0..12]: [311, 308, 313, 309, 310, 303, 284, 310, 310, 305, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466738334Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185402024-02-13T21:46:41.782Z{"msg":"[1]R generation[0..12]: [170, 166, 171, 167, 168, 163, 151, 168, 168, 165, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466758876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185412024-02-13T21:46:41.782Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466773231Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185422024-02-13T21:46:41.782Z{"msg":"[2]R flush_numbers[0..12]: [311, 308, 313, 309, 310, 303, 284, 310, 310, 305, 313, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466795423Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185432024-02-13T21:46:41.782Z{"msg":"[2]R generation[0..12]: [170, 166, 171, 167, 168, 163, 151, 168, 168, 165, 171, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466809608Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185442024-02-13T21:46:41.782Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466823772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185452024-02-13T21:46:41.782Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466837198Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185462024-02-13T21:46:41.782Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
185472024-02-13T21:46:41.782Ztime":"2024-02-13T21:46:40.46685771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185482024-02-13T21:46:41.782Z{"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466873704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185492024-02-13T21:46:41.782Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.46688695Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"":"downstairs","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185502024-02-13T21:46:41.782Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466900495Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185512024-02-13T21:46:41.782Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466919918Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"0","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185522024-02-13T21:46:41.782Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466934212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"1","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185532024-02-13T21:46:41.782Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466947578Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"client":"2","":"downstairs","session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185542024-02-13T21:46:41.783Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466961033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185552024-02-13T21:46:41.783Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466980586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185562024-02-13T21:46:41.783Z{"msg":"e58f4af2-7ad3-405f-a8c0-4d099c348dfa is now active with session: 0d019e4f-73ae-4621-b67e-1c086bf8fa97","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.466993701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185572024-02-13T21:46:41.783Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.467006206Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826,"session_id":"0d019e4f-73ae-4621-b67e-1c086bf8fa97"}
185582024-02-13T21:46:41.783Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:40.467027598Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826}
185592024-02-13T21:46:41.783ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185602024-02-13T21:46:41.783ZLoading write count information from file "/tmp/test_repair_verify.data"
185612024-02-13T21:46:41.783ZRun Repair workload
185622024-02-13T21:46:41.783Z01/30 Write block 369 len 16384 data: 16 15 17 20
185632024-02-13T21:46:41.786Z02/30 Write block 59 len 8192 data: 20 17
185642024-02-13T21:46:41.786Z03/30 Write block 110 len 16384 data: 9 12 11 11
185652024-02-13T21:46:41.786Z04/30 Write block 11 len 32768 data: 15 15 15 16 16 16 15 16
185662024-02-13T21:46:41.786Z05/30 Read block 285 len 20480
185672024-02-13T21:46:41.786Z06/30 Read block 62 len 36864
185682024-02-13T21:46:41.786Z07/30 Flush
185692024-02-13T21:46:41.789Z08/30 Flush
185702024-02-13T21:46:41.789Z09/30 Write block 245 len 8192 data: 9 9
185712024-02-13T21:46:41.789Z10/30 Read block 439 len 24576
185722024-02-13T21:46:41.792Z11/30 Write block 468 len 40960 data: 13 14 14 16 18 19 20 18 15 13
185732024-02-13T21:46:41.792Z12/30 Read block 271 len 4096
185742024-02-13T21:46:41.792Z13/30 Read block 151 len 24576
185752024-02-13T21:46:41.796Z14/30 Read block 92 len 32768
185762024-02-13T21:46:41.796Z15/30 Write block 211 len 20480 data: 10 11 12 13 16
185772024-02-13T21:46:41.796Z16/30 Read block 38 len 36864
185782024-02-13T21:46:41.799Z17/30 Write block 171 len 8192 data: 6 10
185792024-02-13T21:46:41.799Z18/30 Write block 382 len 28672 data: 15 14 14 14 15 14 14
185802024-02-13T21:46:41.799Z19/30 Write block 220 len 40960 data: 20 18 20 21 19 17 14 16 15 12
185812024-02-13T21:46:41.799Z20/30 Read block 437 len 24576
185822024-02-13T21:46:41.802Z21/30 Read block 363 len 12288
185832024-02-13T21:46:41.802Z22/30 Read block 519 len 16384
185842024-02-13T21:46:41.802Z23/30 Write block 48 len 8192 data: 11 11
185852024-02-13T21:46:41.802Z24/30 Read block 33 len 16384
185862024-02-13T21:46:41.805Z25/30 Read block 115 len 32768
185872024-02-13T21:46:41.805Z26/30 Write block 521 len 12288 data: 20 18 17
185882024-02-13T21:46:41.805Z27/30 Write block 294 len 12288 data: 15 13 11
185892024-02-13T21:46:41.805Z28/30 Read block 443 len 4096
185902024-02-13T21:46:41.808Z29/30 Write block 461 len 12288 data: 18 16 14
185912024-02-13T21:46:41.808Z30/30 Read block 65 len 28672
185922024-02-13T21:46:41.809Z----------------------------------------------------------------
185932024-02-13T21:46:41.809Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185942024-02-13T21:46:41.809ZStates: Active Active Active
185952024-02-13T21:46:41.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185962024-02-13T21:46:41.809Z 1 Acked 1000 Write 4 Sent Done Done false
185972024-02-13T21:46:41.809Z 2 Acked 1001 Write 2 Sent Done Done false
185982024-02-13T21:46:41.809Z 3 Acked 1002 Write 4 Sent Done Done false
185992024-02-13T21:46:41.809Z 4 Acked 1003 Write 8 Sent Done Done false
186002024-02-13T21:46:41.809Z 5 Acked 1004 Read 5 Sent Done Done false
186012024-02-13T21:46:41.809Z 6 Acked 1005 Read 9 Sent Done Done false
186022024-02-13T21:46:41.809Z 7 Acked 1006 Flush 0 Sent Done Done false
186032024-02-13T21:46:41.809Z 8 Acked 1007 Flush 0 Sent Done Done false
186042024-02-13T21:46:41.809Z 9 Acked 1008 Write 2 Sent Done Done false
186052024-02-13T21:46:41.809Z 10 Acked 1009 Read 6 Sent Done Done false
186062024-02-13T21:46:41.809Z 11 Acked 1010 Write 10 Sent Done Done false
186072024-02-13T21:46:41.809Z 12 Acked 1011 Read 1 Sent Done Done false
186082024-02-13T21:46:41.809Z 13 Acked 1012 Read 6 Sent Done Done false
186092024-02-13T21:46:41.809Z 14 Acked 1013 Read 8 Sent Done Done false
186102024-02-13T21:46:41.809Z 15 Acked 1014 Write 5 Sent Done Done false
186112024-02-13T21:46:41.809Z 16 Acked 1015 Read 9 Sent Done Done false
186122024-02-13T21:46:41.809Z 17 Acked 1016 Write 2 Sent Done Done false
186132024-02-13T21:46:41.809Z 18 Acked 1017 Write 7 Sent Done Done false
186142024-02-13T21:46:41.809Z 19 Acked 1018 Write 10 Sent Done Done false
186152024-02-13T21:46:41.809Z 20 Acked 1019 Read 6 Sent Done Done false
186162024-02-13T21:46:41.810Z 21 Acked 1020 Read 3 Sent Done Done false
186172024-02-13T21:46:41.810Z 22 Acked 1021 Read 4 Sent Done Done false
186182024-02-13T21:46:41.810Z 23 Acked 1022 Write 2 Sent Done Done false
186192024-02-13T21:46:41.810Z 24 Acked 1023 Read 4 Sent Done Done false
186202024-02-13T21:46:41.810Z 25 Acked 1024 Read 8 Sent Done Done false
186212024-02-13T21:46:41.810Z 26 Acked 1025 Write 3 Sent Done Done false
186222024-02-13T21:46:41.810Z 27 Acked 1026 Write 3 Sent Done Done false
186232024-02-13T21:46:41.810Z 28 Acked 1027 Read 1 Sent Done Done false
186242024-02-13T21:46:41.810Z 29 Acked 1028 Write 3 Sent Done Done false
186252024-02-13T21:46:41.810Z 30 Acked 1029 Read 7 Sent Done Sent false
186262024-02-13T21:46:41.810Z STATES DS:0 DS:1 DS:2 TOTAL
186272024-02-13T21:46:41.810Z New 0 0 0 0
186282024-02-13T21:46:41.810Z Sent 30 0 1 31
186292024-02-13T21:46:41.810Z Done 0 30 29 59
186302024-02-13T21:46:41.810Z Skipped 0 0 0 0
186312024-02-13T21:46:41.810Z Error 0 0 0 0
186322024-02-13T21:46:41.810ZLast Flush: 0 1007 1007
186332024-02-13T21:46:41.810ZDownstairs last five completed:
186342024-02-13T21:46:41.810ZUpstairs last five completed: 30 29 28 27 26
186352024-02-13T21:46:41.810Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:40.495454097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1826}
186362024-02-13T21:46:41.810ZWrote out file "/tmp/test_repair_verify.data"
186372024-02-13T21:46:41.814Z
186382024-02-13T21:46:41.817ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186392024-02-13T21:46:41.833ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186402024-02-13T21:46:41.833Z 0 000-019 170 172 172 311 314 314 F F F <---
186412024-02-13T21:46:41.833Z 1 020-039 166 166 166 308 308 308 F F F
186422024-02-13T21:46:41.833Z 2 040-059 171 172 172 313 314 314 F T T <---
186432024-02-13T21:46:41.833Z 3 060-079 167 172 172 309 314 314 F F F <---
186442024-02-13T21:46:41.833Z 4 080-099 168 168 168 310 310 310 F F F
186452024-02-13T21:46:41.833Z 5 100-119 163 172 172 303 314 314 F F F <---
186462024-02-13T21:46:41.833Z 6 120-139 151 151 151 284 284 284 F F F
186472024-02-13T21:46:41.833Z 7 140-159 168 168 168 310 310 310 F F F
186482024-02-13T21:46:41.837Z 8 160-179 168 168 168 310 310 310 F T T <---
186492024-02-13T21:46:41.837Z 9 180-199 165 165 165 305 305 305 F F F
186502024-02-13T21:46:41.837Z 10 200-219 171 171 171 313 313 313 F T T <---
186512024-02-13T21:46:41.837Z 11 220-239 171 171 171 313 313 313 F T T <---
186522024-02-13T21:46:41.837Z 12 240-259 163 163 163 303 303 303 F T T <---
186532024-02-13T21:46:41.837Z 13 260-279 163 163 163 303 303 303 F F F
186542024-02-13T21:46:41.837Z 14 280-299 168 168 168 310 310 310 F T T <---
186552024-02-13T21:46:41.837Z 15 300-319 158 158 158 297 297 297 F F F
186562024-02-13T21:46:41.837Z 16 320-339 170 170 170 311 311 311 F F F
186572024-02-13T21:46:41.837Z 17 340-359 168 168 168 310 310 310 F F F
186582024-02-13T21:46:41.837Z 18 360-379 166 172 172 307 314 314 F F F <---
186592024-02-13T21:46:41.837Z 19 380-399 166 166 166 307 307 307 F T T <---
186602024-02-13T21:46:41.837Z 20 400-419 167 167 167 309 309 309 F F F
186612024-02-13T21:46:41.837Z 21 420-439 168 168 168 310 310 310 F F F
186622024-02-13T21:46:41.837Z 22 440-459 170 170 170 312 312 312 F F F
186632024-02-13T21:46:41.837Z 23 460-479 170 170 170 312 312 312 F T T <---
186642024-02-13T21:46:41.838Z 24 480-499 170 170 170 311 311 311 F F F
186652024-02-13T21:46:41.838Z 25 500-519 170 170 170 311 311 311 F F F
186662024-02-13T21:46:41.838Z 26 520-539 168 168 168 310 310 310 F T T <---
186672024-02-13T21:46:41.838Z 27 540-559 170 170 170 311 311 311 F F F
186682024-02-13T21:46:41.838Z 28 560-579 171 171 171 313 313 313 F F F
186692024-02-13T21:46:41.838Z 29 580-599 154 154 154 290 290 290 F F F
186702024-02-13T21:46:41.838ZMax gen: 172, Max flush: 314
186712024-02-13T21:46:41.838ZError: Difference in extent metadata found!
186722024-02-13T21:46:41.838ZA Difference in extent metadata is expected here
186732024-02-13T21:46:41.838ZOn loop 86
186742024-02-13T21:46:41.838Z
186752024-02-13T21:46:41.844ZVerifying data now
186762024-02-13T21:46:46.461ZNo change after verify
186772024-02-13T21:46:46.461ZLoop: 86 Downstairs dump after verify (and repair):
186782024-02-13T21:46:46.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186792024-02-13T21:46:46.476Z 0 000-019 172 172 172 314 314 314 F F F
186802024-02-13T21:46:46.476Z 1 020-039 166 166 166 308 308 308 F F F
186812024-02-13T21:46:46.476Z 2 040-059 173 173 173 315 315 315 F F F
186822024-02-13T21:46:46.476Z 3 060-079 172 172 172 314 314 314 F F F
186832024-02-13T21:46:46.476Z 4 080-099 168 168 168 310 310 310 F F F
186842024-02-13T21:46:46.476Z 5 100-119 172 172 172 314 314 314 F F F
186852024-02-13T21:46:46.476Z 6 120-139 151 151 151 284 284 284 F F F
186862024-02-13T21:46:46.476Z 7 140-159 168 168 168 310 310 310 F F F
186872024-02-13T21:46:46.476Z 8 160-179 173 173 173 315 315 315 F F F
186882024-02-13T21:46:46.480Z 9 180-199 165 165 165 305 305 305 F F F
186892024-02-13T21:46:46.480Z 10 200-219 173 173 173 315 315 315 F F F
186902024-02-13T21:46:46.480Z 11 220-239 173 173 173 315 315 315 F F F
186912024-02-13T21:46:46.480Z 12 240-259 173 173 173 315 315 315 F F F
186922024-02-13T21:46:46.480Z 13 260-279 163 163 163 303 303 303 F F F
186932024-02-13T21:46:46.480Z 14 280-299 173 173 173 315 315 315 F F F
186942024-02-13T21:46:46.480Z 15 300-319 158 158 158 297 297 297 F F F
186952024-02-13T21:46:46.480Z 16 320-339 170 170 170 311 311 311 F F F
186962024-02-13T21:46:46.480Z 17 340-359 168 168 168 310 310 310 F F F
186972024-02-13T21:46:46.480Z 18 360-379 172 172 172 314 314 314 F F F
186982024-02-13T21:46:46.480Z 19 380-399 173 173 173 315 315 315 F F F
186992024-02-13T21:46:46.480Z 20 400-419 167 167 167 309 309 309 F F F
187002024-02-13T21:46:46.480Z 21 420-439 168 168 168 310 310 310 F F F
187012024-02-13T21:46:46.480Z 22 440-459 170 170 170 312 312 312 F F F
187022024-02-13T21:46:46.480Z 23 460-479 173 173 173 315 315 315 F F F
187032024-02-13T21:46:46.480Z 24 480-499 170 170 170 311 311 311 F F F
187042024-02-13T21:46:46.480Z 25 500-519 170 170 170 311 311 311 F F F
187052024-02-13T21:46:46.480Z 26 520-539 173 173 173 315 315 315 F F F
187062024-02-13T21:46:46.480Z 27 540-559 170 170 170 311 311 311 F F F
187072024-02-13T21:46:46.480Z 28 560-579 171 171 171 313 313 313 F F F
187082024-02-13T21:46:46.480Z 29 580-599 154 154 154 290 290 290 F F F
187092024-02-13T21:46:46.481ZMax gen: 173, Max flush: 315
187102024-02-13T21:46:46.481Z
187112024-02-13T21:46:46.481ZBegin loop 87
187122024-02-13T21:46:46.481ZDownstairs to restart: 1
187132024-02-13T21:46:46.497Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183074928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
187142024-02-13T21:46:46.504Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183329176Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
187152024-02-13T21:46:46.504Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183356486Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
187162024-02-13T21:46:46.504Z{"msg":"Crucible b240cc59-b5b7-4227-ad55-3791fc76d7df has session id: ed167468-4f5b-4d86-9b04-a43fd222f479","v":0,"name":"crucible","level":30Crucible runtime is spawned
187172024-02-13T21:46:46.504Z,"time":"2024-02-13T21:46:45.183379057Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187182024-02-13T21:46:46.504Z{"msg":"Upstairs opts: Upstairs UUID: b240cc59-b5b7-4227-ad55-3791fc76d7df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183417033Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187192024-02-13T21:46:46.504Z{"msg":"Crucible stats registered with UUID: b240cc59-b5b7-4227-ad55-3791fc76d7df","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183443184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187202024-02-13T21:46:46.504Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183467405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
187212024-02-13T21:46:46.504Z{"msg":"b240cc59-b5b7-4227-ad55-3791fc76d7df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183522086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187222024-02-13T21:46:46.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183545867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187232024-02-13T21:46:46.504Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183560781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187242024-02-13T21:46:46.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183574766Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187252024-02-13T21:46:46.504Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183596619Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187262024-02-13T21:46:46.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183614042Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187272024-02-13T21:46:46.504Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.1836468Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187282024-02-13T21:46:46.504Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183662724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187292024-02-13T21:46:46.504Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183695503Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187302024-02-13T21:46:46.504Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183710927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187312024-02-13T21:46:46.504Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183870479Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187322024-02-13T21:46:46.504Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183910625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187332024-02-13T21:46:46.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.183937225Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"io task","client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187342024-02-13T21:46:46.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.184659073Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187352024-02-13T21:46:46.505Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.184708835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187362024-02-13T21:46:46.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.184758067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187372024-02-13T21:46:46.505Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.184783958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187382024-02-13T21:46:46.505Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185220889Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187392024-02-13T21:46:46.505Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.18532911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187402024-02-13T21:46:46.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185539854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187412024-02-13T21:46:46.505Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185648394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187422024-02-13T21:46:46.505Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185695367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187432024-02-13T21:46:46.505Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185722417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187442024-02-13T21:46:46.505Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185754356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187452024-02-13T21:46:46.505Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.18576923Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187462024-02-13T21:46:46.505Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185790433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187472024-02-13T21:46:46.505Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.185812285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187482024-02-13T21:46:46.505Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.18607486Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187492024-02-13T21:46:46.509Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186476074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187502024-02-13T21:46:46.509Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186505623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187512024-02-13T21:46:46.509Z{"msg":"[0]R flush_numbers[0..12]: [314, 308, 315, 314, 310, 314, 284, 310, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186523817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187522024-02-13T21:46:46.509Z{"msg":"[0]R generation[0..12]: [172, 166, 173, 172, 168, 172, 151, 168, 173, 165, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186558764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187532024-02-13T21:46:46.509Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186576358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187542024-02-13T21:46:46.509Z{"msg":"[1]R flush_numbers[0..12]: [314, 308, 315, 314, 310, 314, 284, 310, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.18659811Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187552024-02-13T21:46:46.510Z{"msg":"[1]R generation[0..12]: [172, 166, 173, 172, 168, 172, 151, 168, 173, 165, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186612434Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187562024-02-13T21:46:46.510Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186638635Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187572024-02-13T21:46:46.510Z{"msg":"[2]R flush_numbers[0..12]: [314, 308, 315, 314, 310, 314, 284, 310, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.18665323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187582024-02-13T21:46:46.510Z{"msg":"[2]R generation[0..12]: [172, 166, 173, 172, 168, 172, 151, 168, 173, 165, 173, 173]"Wait for a query_work_queue command to finish before sending IO
187592024-02-13T21:46:46.510Z,"v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186677121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187602024-02-13T21:46:46.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186692016Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187612024-02-13T21:46:46.510Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186712908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187622024-02-13T21:46:46.510Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186726803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187632024-02-13T21:46:46.510Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186746146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187642024-02-13T21:46:46.510Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186759071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"":"downstairs","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187652024-02-13T21:46:46.510Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186772726Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187662024-02-13T21:46:46.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186784622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"0","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187672024-02-13T21:46:46.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186811532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"1","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187682024-02-13T21:46:46.510Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186825797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"client":"2","":"downstairs","session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187692024-02-13T21:46:46.510Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186840262Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187702024-02-13T21:46:46.510Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186853497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187712024-02-13T21:46:46.510Z{"msg":"b240cc59-b5b7-4227-ad55-3791fc76d7df is now active with session: ed167468-4f5b-4d86-9b04-a43fd222f479","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186867152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187722024-02-13T21:46:46.510Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186879208Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834,"session_id":"ed167468-4f5b-4d86-9b04-a43fd222f479"}
187732024-02-13T21:46:46.510Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:45.186898361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
187742024-02-13T21:46:46.510ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187752024-02-13T21:46:46.510ZLoading write count information from file "/tmp/test_repair_verify.data"
187762024-02-13T21:46:46.510ZRun Repair workload
187772024-02-13T21:46:46.510Z01/30 Write block 144 len 24576 data: 18 16 12 12 11 11
187782024-02-13T21:46:46.510Z02/30 Write block 95 len 20480 data: 10 11 12 11 9
187792024-02-13T21:46:46.511Z03/30 Read block 411 len 24576
187802024-02-13T21:46:46.511Z04/30 Read block 48 len 12288
187812024-02-13T21:46:46.511Z05/30 Read block 404 len 12288
187822024-02-13T21:46:46.511Z06/30 Write block 531 len 40960 data: 14 15 14 15 15 13 14 17 20 23
187832024-02-13T21:46:46.511Z07/30 Read block 228 len 40960
187842024-02-13T21:46:46.514Z08/30 Read block 84 len 36864
187852024-02-13T21:46:46.514Z09/30 Read block 197 len 40960
187862024-02-13T21:46:46.515Z10/30 Read block 574 len 4096
187872024-02-13T21:46:46.515Z11/30 Write block 383 len 12288 data: 15 15 15
187882024-02-13T21:46:46.515Z12/30 Read block 75 len 8192
187892024-02-13T21:46:46.515Z13/30 Write block 580 len 24576 data: 6 7 8 10 11 12
187902024-02-13T21:46:46.515Z14/30 Flush
187912024-02-13T21:46:46.518Z15/30 Write block 426 len 16384 data: 12 13 15 19
187922024-02-13T21:46:46.518Z16/30 Write block 339 len 40960 data: 10 13 13 14 16 14 13 12 13 13
187932024-02-13T21:46:46.521Z17/30 Write block 502 len 4096 data: 10
187942024-02-13T21:46:46.521Z18/30 Read block 4 len 16384
187952024-02-13T21:46:46.521Z19/30 Flush
187962024-02-13T21:46:46.524Z20/30 Write block 407 len 12288 data: 12 15 17
187972024-02-13T21:46:46.524Z21/30 Write block 505 len 8192 data: 9 13
187982024-02-13T21:46:46.524Z22/30 Read block 487 len 20480
187992024-02-13T21:46:46.524Z23/30 Read block 216 len 20480
188002024-02-13T21:46:46.527Z24/30 Read block 533 len 20480
188012024-02-13T21:46:46.527Z25/30 Write block 23 len 16384 data: 15 15 15 13
188022024-02-13T21:46:46.527Z26/30 Read block 281 len 12288
188032024-02-13T21:46:46.527Z27/30 Write block 36 len 12288 data: 16 12 13
188042024-02-13T21:46:46.531Z28/30 Write block 491 len 8192 data: 10 10
188052024-02-13T21:46:46.531Z29/30 Read block 308 len 28672
188062024-02-13T21:46:46.531Z30/30 Write block 543 len 4096 data: 18
188072024-02-13T21:46:46.531Z----------------------------------------------------------------
188082024-02-13T21:46:46.531Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
188092024-02-13T21:46:46.531ZStates: Active Active Active
188102024-02-13T21:46:46.531ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188112024-02-13T21:46:46.531Z 1 Acked 1000 Write 6 Done Sent Done false
188122024-02-13T21:46:46.531Z 2 Acked 1001 Write 5 Done Sent Done false
188132024-02-13T21:46:46.531Z 3 Acked 1002 Read 6 Done Sent Done false
188142024-02-13T21:46:46.531Z 4 Acked 1003 Read 3 Done Sent Done false
188152024-02-13T21:46:46.531Z 5 Acked 1004 Read 3 Done Sent Done false
188162024-02-13T21:46:46.531Z 6 Acked 1005 Write 10 Done Sent Done false
188172024-02-13T21:46:46.531Z 7 Acked 1006 Read 10 Done Sent Done false
188182024-02-13T21:46:46.531Z 8 Acked 1007 Read 9 Done Sent Done false
188192024-02-13T21:46:46.531Z 9 Acked 1008 Read 10 Done Sent Done false
188202024-02-13T21:46:46.531Z 10 Acked 1009 Read 1 Done Sent Done false
188212024-02-13T21:46:46.531Z 11 Acked 1010 Write 3 Done Sent Done false
188222024-02-13T21:46:46.531Z 12 Acked 1011 Read 2 Done Sent Done false
188232024-02-13T21:46:46.531Z 13 Acked 1012 Write 6 Done Sent Done false
188242024-02-13T21:46:46.531Z 14 Acked 1013 Flush 0 Done Sent Done false
188252024-02-13T21:46:46.531Z 15 Acked 1014 Write 4 Done Sent Done false
188262024-02-13T21:46:46.532Z 16 Acked 1015 Write 10 Done Sent Done false
188272024-02-13T21:46:46.532Z 17 Acked 1016 Write 1 Done Sent Done false
188282024-02-13T21:46:46.532Z 18 Acked 1017 Read 4 Done Sent Done false
188292024-02-13T21:46:46.532Z 19 Acked 1018 Flush 0 Done Sent Done false
188302024-02-13T21:46:46.532Z 20 Acked 1019 Write 3 Done Sent Done false
188312024-02-13T21:46:46.532Z 21 Acked 1020 Write 2 Done Sent Done false
188322024-02-13T21:46:46.532Z 22 Acked 1021 Read 5 Done Sent Done false
188332024-02-13T21:46:46.532Z 23 Acked 1022 Read 5 Done Sent Done false
188342024-02-13T21:46:46.532Z 24 Acked 1023 Read 5 Done Sent Done false
188352024-02-13T21:46:46.532Z 25 Acked 1024 Write 4 Done Sent Done false
188362024-02-13T21:46:46.532Z 26 Acked 1025 Read 3 Done Sent Done false
188372024-02-13T21:46:46.532Z 27 Acked 1026 Write 3 Done Sent Done false
188382024-02-13T21:46:46.532Z 28 Acked 1027 Write 2 Done Sent Done false
188392024-02-13T21:46:46.532Z 29 Acked 1028 Read 7 Done Sent Done false
188402024-02-13T21:46:46.532Z 30 Acked 1029 Write 1 Sent Sent Sent false
188412024-02-13T21:46:46.532Z STATES DS:0 DS:1 DS:2 TOTAL
188422024-02-13T21:46:46.532Z New 0 0 0 0
188432024-02-13T21:46:46.532Z Sent 1 30 1 32
188442024-02-13T21:46:46.532Z Done 29 0 29 58
188452024-02-13T21:46:46.532Z Skipped 0 0 0 0
188462024-02-13T21:46:46.532Z Error 0 0 0 0
188472024-02-13T21:46:46.532ZLast Flush: 1018 0 1018
188482024-02-13T21:46:46.532ZDownstairs last five completed:
188492024-02-13T21:46:46.532ZUpstairs last five completed: 30 29 28 27 26
188502024-02-13T21:46:46.532Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:45.215427292Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1834}
188512024-02-13T21:46:46.532ZWrote out file "/tmp/test_repair_verify.data"
188522024-02-13T21:46:46.536Z
188532024-02-13T21:46:46.539ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188542024-02-13T21:46:46.553ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188552024-02-13T21:46:46.553Z 0 000-019 172 172 172 314 314 314 F F F
188562024-02-13T21:46:46.553Z 1 020-039 166 166 166 308 308 308 T F T <---
188572024-02-13T21:46:46.553Z 2 040-059 173 173 173 315 315 315 F F F
188582024-02-13T21:46:46.553Z 3 060-079 172 172 172 314 314 314 F F F
188592024-02-13T21:46:46.553Z 4 080-099 174 168 174 316 310 316 F F F <---
188602024-02-13T21:46:46.553Z 5 100-119 172 172 172 314 314 314 F F F
188612024-02-13T21:46:46.553Z 6 120-139 151 151 151 284 284 284 F F F
188622024-02-13T21:46:46.553Z 7 140-159 174 168 174 316 310 316 F F F <---
188632024-02-13T21:46:46.553Z 8 160-179 173 173 173 315 315 315 F F F
188642024-02-13T21:46:46.553Z 9 180-199 165 165 165 305 305 305 F F F
188652024-02-13T21:46:46.553Z 10 200-219 173 173 173 315 315 315 F F F
188662024-02-13T21:46:46.553Z 11 220-239 173 173 173 315 315 315 F F F
188672024-02-13T21:46:46.553Z 12 240-259 173 173 173 315 315 315 F F F
188682024-02-13T21:46:46.553Z 13 260-279 163 163 163 303 303 303 F F F
188692024-02-13T21:46:46.553Z 14 280-299 173 173 173 315 315 315 F F F
188702024-02-13T21:46:46.557Z 15 300-319 158 158 158 297 297 297 F F F
188712024-02-13T21:46:46.557Z 16 320-339 174 170 174 317 311 317 F F F <---
188722024-02-13T21:46:46.557Z 17 340-359 174 168 174 317 310 317 F F F <---
188732024-02-13T21:46:46.557Z 18 360-379 172 172 172 314 314 314 F F F
188742024-02-13T21:46:46.557Z 19 380-399 174 173 174 316 315 316 F F F <---
188752024-02-13T21:46:46.557Z 20 400-419 167 167 167 309 309 309 T F T <---
188762024-02-13T21:46:46.557Z 21 420-439 174 168 174 317 310 317 F F F <---
188772024-02-13T21:46:46.557Z 22 440-459 170 170 170 312 312 312 F F F
188782024-02-13T21:46:46.557Z 23 460-479 173 173 173 315 315 315 F F F
188792024-02-13T21:46:46.557Z 24 480-499 170 170 170 311 311 311 T F T <---
188802024-02-13T21:46:46.557Z 25 500-519 174 170 174 317 311 317 T F T <---
188812024-02-13T21:46:46.557Z 26 520-539 174 173 174 316 315 316 F F F <---
188822024-02-13T21:46:46.557Z 27 540-559 174 170 174 316 311 316 T F T <---
188832024-02-13T21:46:46.557Z 28 560-579 171 171 171 313 313 313 F F F
188842024-02-13T21:46:46.557Z 29 580-599 174 154 174 316 290 316 F F F <---
188852024-02-13T21:46:46.557ZMax gen: 174, Max flush: 317
188862024-02-13T21:46:46.557ZError: Difference in extent metadata found!
188872024-02-13T21:46:46.557ZA Difference in extent metadata is expected here
188882024-02-13T21:46:46.557ZOn loop 87
188892024-02-13T21:46:46.557Z
188902024-02-13T21:46:46.560ZVerifying data now
188912024-02-13T21:46:51.176ZNo change after verify
188922024-02-13T21:46:51.177ZLoop: 87 Downstairs dump after verify (and repair):
188932024-02-13T21:46:51.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188942024-02-13T21:46:51.188Z 0 000-019 172 172 172 314 314 314 F F F
188952024-02-13T21:46:51.188Z 1 020-039 175 175 175 318 318 318 F F F
188962024-02-13T21:46:51.188Z 2 040-059 173 173 173 315 315 315 F F F
188972024-02-13T21:46:51.188Z 3 060-079 172 172 172 314 314 314 F F F
188982024-02-13T21:46:51.189Z 4 080-099 174 174 174 316 316 316 F F F
188992024-02-13T21:46:51.189Z 5 100-119 172 172 172 314 314 314 F F F
189002024-02-13T21:46:51.189Z 6 120-139 151 151 151 284 284 284 F F F
189012024-02-13T21:46:51.189Z 7 140-159 174 174 174 316 316 316 F F F
189022024-02-13T21:46:51.189Z 8 160-179 173 173 173 315 315 315 F F F
189032024-02-13T21:46:51.192Z 9 180-199 165 165 165 305 305 305 F F F
189042024-02-13T21:46:51.192Z 10 200-219 173 173 173 315 315 315 F F F
189052024-02-13T21:46:51.192Z 11 220-239 173 173 173 315 315 315 F F F
189062024-02-13T21:46:51.192Z 12 240-259 173 173 173 315 315 315 F F F
189072024-02-13T21:46:51.192Z 13 260-279 163 163 163 303 303 303 F F F
189082024-02-13T21:46:51.193Z 14 280-299 173 173 173 315 315 315 F F F
189092024-02-13T21:46:51.193Z 15 300-319 158 158 158 297 297 297 F F F
189102024-02-13T21:46:51.193Z 16 320-339 174 174 174 317 317 317 F F F
189112024-02-13T21:46:51.193Z 17 340-359 174 174 174 317 317 317 F F F
189122024-02-13T21:46:51.193Z 18 360-379 172 172 172 314 314 314 F F F
189132024-02-13T21:46:51.193Z 19 380-399 174 174 174 316 316 316 F F F
189142024-02-13T21:46:51.193Z 20 400-419 175 175 175 318 318 318 F F F
189152024-02-13T21:46:51.193Z 21 420-439 174 174 174 317 317 317 F F F
189162024-02-13T21:46:51.193Z 22 440-459 170 170 170 312 312 312 F F F
189172024-02-13T21:46:51.193Z 23 460-479 173 173 173 315 315 315 F F F
189182024-02-13T21:46:51.193Z 24 480-499 175 175 175 318 318 318 F F F
189192024-02-13T21:46:51.193Z 25 500-519 175 175 175 318 318 318 F F F
189202024-02-13T21:46:51.193Z 26 520-539 174 174 174 316 316 316 F F F
189212024-02-13T21:46:51.193Z 27 540-559 175 175 175 318 318 318 F F F
189222024-02-13T21:46:51.193Z 28 560-579 171 171 171 313 313 313 F F F
189232024-02-13T21:46:51.193Z 29 580-599 174 174 174 316 316 316 F F F
189242024-02-13T21:46:51.193ZMax gen: 175, Max flush: 318
189252024-02-13T21:46:51.193Z
189262024-02-13T21:46:51.193ZBegin loop 88
189272024-02-13T21:46:51.193ZDownstairs to restart: 1
189282024-02-13T21:46:51.209Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893139346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
189292024-02-13T21:46:51.209Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893409069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
189302024-02-13T21:46:51.214Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
189312024-02-13T21:46:51.214Z2024-02-13T21:46:49.893441847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
189322024-02-13T21:46:51.214Z{"msg":"Crucible 1d03d3b6-80d6-44a6-b839-b9f817251c49 has session id: b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893481013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189332024-02-13T21:46:51.214Z{"msg":"Upstairs opts: Upstairs UUID: 1d03d3b6-80d6-44a6-b839-b9f817251c49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893525996Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189342024-02-13T21:46:51.214Z{"msg":"Crucible stats registered with UUID: 1d03d3b6-80d6-44a6-b839-b9f817251c49","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893565192Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189352024-02-13T21:46:51.214Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893581466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
189362024-02-13T21:46:51.214Z{"msg":"1d03d3b6-80d6-44a6-b839-b9f817251c49 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893706341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189372024-02-13T21:46:51.214Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893732571Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189382024-02-13T21:46:51.214Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893750075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189392024-02-13T21:46:51.214Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893788691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189402024-02-13T21:46:51.214Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893813872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189412024-02-13T21:46:51.214Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893849389Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189422024-02-13T21:46:51.214Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893877159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189432024-02-13T21:46:51.214Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.89390194Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189442024-02-13T21:46:51.214Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.893964067Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189452024-02-13T21:46:51.214Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894001804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189462024-02-13T21:46:51.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894146631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189472024-02-13T21:46:51.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894179669Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189482024-02-13T21:46:51.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894217546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"io task","client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189492024-02-13T21:46:51.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894717184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189502024-02-13T21:46:51.215Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894758189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189512024-02-13T21:46:51.215Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894785329Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189522024-02-13T21:46:51.215Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.894817318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189532024-02-13T21:46:51.215Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.89525255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189542024-02-13T21:46:51.215Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895368388Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189552024-02-13T21:46:51.215Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895572454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189562024-02-13T21:46:51.215Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895608961Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189572024-02-13T21:46:51.215Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895810877Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189582024-02-13T21:46:51.215Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.8958586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189592024-02-13T21:46:51.215Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.8959148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189602024-02-13T21:46:51.215Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895934992Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189612024-02-13T21:46:51.215Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895963522Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189622024-02-13T21:46:51.215Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.895978527Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189632024-02-13T21:46:51.215Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896103491Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189642024-02-13T21:46:51.215Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896505196Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189652024-02-13T21:46:51.215Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896538673Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189662024-02-13T21:46:51.215Z{"msg":"[0]R flush_numbers[0..12]: [314, 318, 315, 314, 316, 314, 284, 316, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896554378Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189672024-02-13T21:46:51.215Z{"msg":"[0]R generation[0..12]: [172, 175, 173, 172, 174, 172, 151, 174, 173, 165, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.89657672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189682024-02-13T21:46:51.219Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896592144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189692024-02-13T21:46:51.219Z{"msg":"[1]R flush_numbers[0..12]: [314, 318, 315, 314, 316, 314, 284, 316, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896607428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189702024-02-13T21:46:51.219Z{"msg":"[1]R generation[0..12]: [172, 175, 173, 172, 174, 172, 151, 174, 173, 165, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896621474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189712024-02-13T21:46:51.219Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896642476Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189722024-02-13T21:46:51.220Z{"msg":"[2]R flush_numbers[0..12]: [314, 318, 315, 314, 316, 314, 284, 316, 315, 305, 315, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.89665771Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189732024-02-13T21:46:51.220Z{"msg":"[2]R generation[0..12]: [172, 175, 173, 172, 174, 172, 151, 174, 173, 165, 173, 173]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189742024-02-13T21:46:51.220Z,"time":"2024-02-13T21:46:49.896691908Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189752024-02-13T21:46:51.220Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896721957Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189762024-02-13T21:46:51.220Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896753575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189772024-02-13T21:46:51.220Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.89676911Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189782024-02-13T21:46:51.220Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896789193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189792024-02-13T21:46:51.220Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896809145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"":"downstairs","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189802024-02-13T21:46:51.220Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896845962Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189812024-02-13T21:46:51.220Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896866645Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"0","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189822024-02-13T21:46:51.220Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896889236Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"1","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189832024-02-13T21:46:51.220Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896922784Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"client":"2","":"downstairs","session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189842024-02-13T21:46:51.220Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896957951Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189852024-02-13T21:46:51.220Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.896982912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189862024-02-13T21:46:51.220Z{"msg":"1d03d3b6-80d6-44a6-b839-b9f817251c49 is now active with session: b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.897000676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189872024-02-13T21:46:51.220Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.897033144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842,"session_id":"b3444a2f-d60d-40c4-b4ac-ff6d2ece80c1"}
189882024-02-13T21:46:51.220Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:49.897064273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
189892024-02-13T21:46:51.220ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189902024-02-13T21:46:51.220ZLoading write count information from file "/tmp/test_repair_verify.data"
189912024-02-13T21:46:51.220ZRun Repair workload
189922024-02-13T21:46:51.220Z01/30 Write block 30 len 4096 data: 11
189932024-02-13T21:46:51.220Z02/30 Read block 240 len 4096
189942024-02-13T21:46:51.220Z03/30 Flush
189952024-02-13T21:46:51.220Z04/30 Flush
189962024-02-13T21:46:51.220Z05/30 Write block 202 len 8192 data: 12 11
189972024-02-13T21:46:51.221Z06/30 Read block 131 len 32768
189982024-02-13T21:46:51.224Z07/30 Write block 212 len 8192 data: 12 13
189992024-02-13T21:46:51.224Z08/30 Write block 292 len 28672 data: 15 16 16 14 12 13 13
190002024-02-13T21:46:51.224Z09/30 Read block 205 len 20480
190012024-02-13T21:46:51.224Z10/30 Write block 397 len 8192 data: 9 10
190022024-02-13T21:46:51.224Z11/30 Read block 307 len 28672
190032024-02-13T21:46:51.224Z12/30 Read block 576 len 12288
190042024-02-13T21:46:51.231Z13/30 Write block 526 len 36864 data: 10 13 13 12 11 15 16 15 16
190052024-02-13T21:46:51.231Z14/30 Write block 523 len 32768 data: 18 13 11 11 14 14 13 12
190062024-02-13T21:46:51.231Z15/30 Flush
190072024-02-13T21:46:51.231Z16/30 Write block 331 len 20480 data: 21 20 17 13 17
190082024-02-13T21:46:51.231Z17/30 Read block 457 len 12288
190092024-02-13T21:46:51.231Z18/30 Write block 494 len 8192 data: 15 13
190102024-02-13T21:46:51.232Z19/30 Read block 516 len 20480
190112024-02-13T21:46:51.238Z20/30 Read block 510 len 8192
190122024-02-13T21:46:51.238Z21/30 Flush
190132024-02-13T21:46:51.238Z22/30 Write block 66 len 40960 data: 11 13 14 19 18 16 18 16 17 14
190142024-02-13T21:46:51.238Z23/30 Flush
190152024-02-13T21:46:51.242Z24/30 Flush
190162024-02-13T21:46:51.242Z25/30 Write block 149 len 36864 data: 12 10 10 10 13 13 14 12 12
190172024-02-13T21:46:51.242Z26/30 Write block 90 len 28672 data: 12 12 11 10 9 11 12
190182024-02-13T21:46:51.242Z27/30 Write block 183 len 40960 data: 12 13 16 13 12 14 13 12 11 11
190192024-02-13T21:46:51.242Z28/30 Read block 92 len 36864
190202024-02-13T21:46:51.246Z29/30 Write block 493 len 16384 data: 13 16 14 14
190212024-02-13T21:46:51.246Z30/30 Read block 305 len 20480
190222024-02-13T21:46:51.246Z----------------------------------------------------------------
190232024-02-13T21:46:51.246Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
190242024-02-13T21:46:51.246ZStates: Active Active Active
190252024-02-13T21:46:51.246ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190262024-02-13T21:46:51.246Z 1 Acked 1000 Write 1 Done Sent Done false
190272024-02-13T21:46:51.246Z 2 Acked 1001 Read 1 Done Sent Done false
190282024-02-13T21:46:51.246Z 3 Acked 1002 Flush 0 Done Sent Done false
190292024-02-13T21:46:51.246Z 4 Acked 1003 Flush 0 Done Sent Done false
190302024-02-13T21:46:51.246Z 5 Acked 1004 Write 2 Done Sent Done false
190312024-02-13T21:46:51.246Z 6 Acked 1005 Read 8 Done Sent Done false
190322024-02-13T21:46:51.246Z 7 Acked 1006 Write 2 Done Sent Done false
190332024-02-13T21:46:51.246Z 8 Acked 1007 Write 7 Done Sent Done false
190342024-02-13T21:46:51.247Z 9 Acked 1008 Read 5 Done Sent Done false
190352024-02-13T21:46:51.247Z 10 Acked 1009 Write 2 Done Sent Done false
190362024-02-13T21:46:51.247Z 11 Acked 1010 Read 7 Done Sent Done false
190372024-02-13T21:46:51.247Z 12 Acked 1011 Read 3 Done Sent Done false
190382024-02-13T21:46:51.247Z 13 Acked 1012 Write 9 Done Sent Done false
190392024-02-13T21:46:51.247Z 14 Acked 1013 Write 8 Done Sent Done false
190402024-02-13T21:46:51.247Z 15 Acked 1014 Flush 0 Done Sent Done false
190412024-02-13T21:46:51.247Z 16 Acked 1015 Write 5 Done Sent Done false
190422024-02-13T21:46:51.247Z 17 Acked 1016 Read 3 Done Sent Done false
190432024-02-13T21:46:51.247Z 18 Acked 1017 Write 2 Done Sent Done false
190442024-02-13T21:46:51.247Z 19 Acked 1018 Read 5 Done Sent Done false
190452024-02-13T21:46:51.247Z 20 Acked 1019 Read 2 Done Sent Done false
190462024-02-13T21:46:51.247Z 21 Acked 1020 Flush 0 Done Sent Done false
190472024-02-13T21:46:51.247Z 22 Acked 1021 Write 10 Done Sent Done false
190482024-02-13T21:46:51.247Z 23 Acked 1022 Flush 0 Done Sent Done false
190492024-02-13T21:46:51.247Z 24 Acked 1023 Flush 0 Done Sent Done false
190502024-02-13T21:46:51.247Z 25 Acked 1024 Write 9 Done Sent Done false
190512024-02-13T21:46:51.247Z 26 Acked 1025 Write 7 Done Sent Done false
190522024-02-13T21:46:51.247Z 27 Acked 1026 Write 10 Done Sent Done false
190532024-02-13T21:46:51.247Z 28 Acked 1027 Read 9 Done Sent Done false
190542024-02-13T21:46:51.247Z 29 Acked 1028 Write 4 Done Sent Done false
190552024-02-13T21:46:51.247Z 30 Acked 1029 Read 5 Sent Sent Done false
190562024-02-13T21:46:51.247Z STATES DS:0 DS:1 DS:2 TOTAL
190572024-02-13T21:46:51.247Z New 0 0 0 0
190582024-02-13T21:46:51.247Z Sent 1 30 0 31
190592024-02-13T21:46:51.248Z Done 29 0 30 59
190602024-02-13T21:46:51.248Z Skipped 0 0 0 0
190612024-02-13T21:46:51.248Z Error 0 0 0 0
190622024-02-13T21:46:51.248ZLast Flush: 1023 0 1023
190632024-02-13T21:46:51.248ZDownstairs last five completed:
190642024-02-13T21:46:51.248ZUpstairs last five completed: 30 29 28 27 26
190652024-02-13T21:46:51.248Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:49.928701617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1842}
190662024-02-13T21:46:51.248ZWrote out file "/tmp/test_repair_verify.data"
190672024-02-13T21:46:51.251Z
190682024-02-13T21:46:51.251ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190692024-02-13T21:46:51.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190702024-02-13T21:46:51.267Z 0 000-019 172 172 172 314 314 314 F F F
190712024-02-13T21:46:51.267Z 1 020-039 176 175 176 319 318 319 F F F <---
190722024-02-13T21:46:51.267Z 2 040-059 173 173 173 315 315 315 F F F
190732024-02-13T21:46:51.267Z 3 060-079 176 172 176 323 314 323 F F F <---
190742024-02-13T21:46:51.267Z 4 080-099 174 174 174 316 316 316 T F T <---
190752024-02-13T21:46:51.267Z 5 100-119 172 172 172 314 314 314 F F F
190762024-02-13T21:46:51.267Z 6 120-139 151 151 151 284 284 284 F F F
190772024-02-13T21:46:51.267Z 7 140-159 174 174 174 316 316 316 T F T <---
190782024-02-13T21:46:51.273Z 8 160-179 173 173 173 315 315 315 F F F
190792024-02-13T21:46:51.273Z 9 180-199 165 165 165 305 305 305 T F T <---
190802024-02-13T21:46:51.273Z 10 200-219 176 173 176 321 315 321 F F F <---
190812024-02-13T21:46:51.273Z 11 220-239 173 173 173 315 315 315 F F F
190822024-02-13T21:46:51.273Z 12 240-259 173 173 173 315 315 315 F F F
190832024-02-13T21:46:51.273Z 13 260-279 163 163 163 303 303 303 F F F
190842024-02-13T21:46:51.273Z 14 280-299 176 173 176 321 315 321 F F F <---
190852024-02-13T21:46:51.273Z 15 300-319 158 158 158 297 297 297 F F F
190862024-02-13T21:46:51.273Z 16 320-339 176 174 176 322 317 322 F F F <---
190872024-02-13T21:46:51.273Z 17 340-359 174 174 174 317 317 317 F F F
190882024-02-13T21:46:51.273Z 18 360-379 172 172 172 314 314 314 F F F
190892024-02-13T21:46:51.274Z 19 380-399 176 174 176 321 316 321 F F F <---
190902024-02-13T21:46:51.274Z 20 400-419 175 175 175 318 318 318 F F F
190912024-02-13T21:46:51.274Z 21 420-439 174 174 174 317 317 317 F F F
190922024-02-13T21:46:51.274Z 22 440-459 170 170 170 312 312 312 F F F
190932024-02-13T21:46:51.274Z 23 460-479 173 173 173 315 315 315 F F F
190942024-02-13T21:46:51.274Z 24 480-499 176 175 176 322 318 322 T F T <---
190952024-02-13T21:46:51.274Z 25 500-519 175 175 175 318 318 318 F F F
190962024-02-13T21:46:51.274Z 26 520-539 176 174 176 321 316 321 F F F <---
190972024-02-13T21:46:51.274Z 27 540-559 175 175 175 318 318 318 F F F
190982024-02-13T21:46:51.274Z 28 560-579 171 171 171 313 313 313 F F F
190992024-02-13T21:46:51.274Z 29 580-599 174 174 174 316 316 316 F F F
191002024-02-13T21:46:51.274ZMax gen: 176, Max flush: 323
191012024-02-13T21:46:51.274ZError: Difference in extent metadata found!
191022024-02-13T21:46:51.274ZA Difference in extent metadata is expected here
191032024-02-13T21:46:51.274ZOn loop 88
191042024-02-13T21:46:51.274Z
191052024-02-13T21:46:51.278ZVerifying data now
191062024-02-13T21:46:55.809ZNo change after verify
191072024-02-13T21:46:55.809ZLoop: 88 Downstairs dump after verify (and repair):
191082024-02-13T21:46:55.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191092024-02-13T21:46:55.824Z 0 000-019 172 172 172 314 314 314 F F F
191102024-02-13T21:46:55.824Z 1 020-039 176 176 176 319 319 319 F F F
191112024-02-13T21:46:55.824Z 2 040-059 173 173 173 315 315 315 F F F
191122024-02-13T21:46:55.824Z 3 060-079 176 176 176 323 323 323 F F F
191132024-02-13T21:46:55.824Z 4 080-099 177 177 177 324 324 324 F F F
191142024-02-13T21:46:55.824Z 5 100-119 172 172 172 314 314 314 F F F
191152024-02-13T21:46:55.824Z 6 120-139 151 151 151 284 284 284 F F F
191162024-02-13T21:46:55.824Z 7 140-159 177 177 177 324 324 324 F F F
191172024-02-13T21:46:55.824Z 8 160-179 173 173 173 315 315 315 F F F
191182024-02-13T21:46:55.827Z 9 180-199 177 177 177 324 324 324 F F F
191192024-02-13T21:46:55.827Z 10 200-219 176 176 176 321 321 321 F F F
191202024-02-13T21:46:55.828Z 11 220-239 173 173 173 315 315 315 F F F
191212024-02-13T21:46:55.828Z 12 240-259 173 173 173 315 315 315 F F F
191222024-02-13T21:46:55.828Z 13 260-279 163 163 163 303 303 303 F F F
191232024-02-13T21:46:55.828Z 14 280-299 176 176 176 321 321 321 F F F
191242024-02-13T21:46:55.828Z 15 300-319 158 158 158 297 297 297 F F F
191252024-02-13T21:46:55.828Z 16 320-339 176 176 176 322 322 322 F F F
191262024-02-13T21:46:55.828Z 17 340-359 174 174 174 317 317 317 F F F
191272024-02-13T21:46:55.828Z 18 360-379 172 172 172 314 314 314 F F F
191282024-02-13T21:46:55.828Z 19 380-399 176 176 176 321 321 321 F F F
191292024-02-13T21:46:55.828Z 20 400-419 175 175 175 318 318 318 F F F
191302024-02-13T21:46:55.828Z 21 420-439 174 174 174 317 317 317 F F F
191312024-02-13T21:46:55.828Z 22 440-459 170 170 170 312 312 312 F F F
191322024-02-13T21:46:55.828Z 23 460-479 173 173 173 315 315 315 F F F
191332024-02-13T21:46:55.828Z 24 480-499 177 177 177 324 324 324 F F F
191342024-02-13T21:46:55.828Z 25 500-519 175 175 175 318 318 318 F F F
191352024-02-13T21:46:55.828Z 26 520-539 176 176 176 321 321 321 F F F
191362024-02-13T21:46:55.828Z 27 540-559 175 175 175 318 318 318 F F F
191372024-02-13T21:46:55.828Z 28 560-579 171 171 171 313 313 313 F F F
191382024-02-13T21:46:55.828Z 29 580-599 174 174 174 316 316 316 F F F
191392024-02-13T21:46:55.828ZMax gen: 177, Max flush: 324
191402024-02-13T21:46:55.828Z
191412024-02-13T21:46:55.828ZBegin loop 89
191422024-02-13T21:46:55.828ZDownstairs to restart: 0
191432024-02-13T21:46:55.847Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529451548Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
191442024-02-13T21:46:55.847Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529680275Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
191452024-02-13T21:46:55.847Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529697139Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
191462024-02-13T21:46:55.852Z{"msg":"Crucible 7f32dc88-8179-477f-84f3-f53c3ba4cc42 has session id: 26e384d3-c0f4-47ad-8796-95229d1e2cc9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529711804Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191472024-02-13T21:46:55.852Z{"msg":"Upstairs opts: Upstairs UUID: 7f32dc88-8179-477f-84f3-f53c3ba4cc42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529747561Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191482024-02-13T21:46:55.852Z{"msg":"Crucible stats registered with UUID: 7f32dc88-8179-477f-84f3-f53c3ba4cc42","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529771752Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191492024-02-13T21:46:55.853ZCrucible runtime is spawned
191502024-02-13T21:46:55.853Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.529899696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
191512024-02-13T21:46:55.853Z{"msg":"7f32dc88-8179-477f-84f3-f53c3ba4cc42 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.53002658Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191522024-02-13T21:46:55.853Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530055919Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191532024-02-13T21:46:55.853Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530093515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191542024-02-13T21:46:55.853Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530122625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191552024-02-13T21:46:55.853Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530149455Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191562024-02-13T21:46:55.853Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530187991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191572024-02-13T21:46:55.853Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530215841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191582024-02-13T21:46:55.853Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530241362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191592024-02-13T21:46:55.853Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530297891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191602024-02-13T21:46:55.853Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530337667Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191612024-02-13T21:46:55.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530391238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191622024-02-13T21:46:55.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.530430523Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191632024-02-13T21:46:55.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.53046664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"io task","client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191642024-02-13T21:46:55.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531149392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191652024-02-13T21:46:55.853Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531220007Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191662024-02-13T21:46:55.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531290841Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191672024-02-13T21:46:55.853Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531402651Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191682024-02-13T21:46:55.853Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531698803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191692024-02-13T21:46:55.853Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531841141Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191702024-02-13T21:46:55.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531888264Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191712024-02-13T21:46:55.853Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.531940485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191722024-02-13T21:46:55.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532197742Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191732024-02-13T21:46:55.853Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532233509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191742024-02-13T21:46:55.854Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532249573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191752024-02-13T21:46:55.854Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532394931Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191762024-02-13T21:46:55.854Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.53242584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191772024-02-13T21:46:55.854Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532440514Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191782024-02-13T21:46:55.854Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532454239Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191792024-02-13T21:46:55.854Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532854794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191802024-02-13T21:46:55.854Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532886962Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191812024-02-13T21:46:55.854Z{"msg":"[0]R flush_numbers[0..12]: [314, 319, 315, 323, 324, 314, 284, 324, 315, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532902237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191822024-02-13T21:46:55.854Z{"msg":"[0]R generation[0..12]: [172, 176, 173, 176, 177, 172, 151, 177, 173, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532917081Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191832024-02-13T21:46:55.854Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532944112Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191842024-02-13T21:46:55.854Z{"msg":"[1]R flush_numbers[0..12]: [314, 319, 315, 323, 324, 314, 284, 324, 315, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532960146Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191852024-02-13T21:46:55.854Z{"msg":"[1]R generation[0..12]: [172, 176, 173, 176, 177, 172, 151, 177, 173, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532981708Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191862024-02-13T21:46:55.854Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.532996382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191872024-02-13T21:46:55.854Z{"msg":"[2]R flush_numbers[0..12]: [314, 319, 315, 323, 324, 314, 284, 324, 315, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533010797Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191882024-02-13T21:46:55.854Z{"msg":"[2]R generation[0..12]: [172, 176, 173, 176, 177, 172, 151, 177, 173, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533024592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191892024-02-13T21:46:55.854Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533046274Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191902024-02-13T21:46:55.854Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533065177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191912024-02-13T21:46:55.854ZWait for a query_work_queue command to finish before sending IO
191922024-02-13T21:46:55.854Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533102134Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191932024-02-13T21:46:55.854Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533129534Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191942024-02-13T21:46:55.854Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533153296Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"":"downstairs","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191952024-02-13T21:46:55.854Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533167451Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191962024-02-13T21:46:55.854Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533179966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"0","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191972024-02-13T21:46:55.854Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533212984Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"1","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191982024-02-13T21:46:55.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533237915Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"client":"2","":"downstairs","session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
191992024-02-13T21:46:55.855Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533261456Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
192002024-02-13T21:46:55.855Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533283329Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
192012024-02-13T21:46:55.855Z{"msg":"7f32dc88-8179-477f-84f3-f53c3ba4cc42 is now active with session: 26e384d3-c0f4-47ad-8796-95229d1e2cc9","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533315687Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
192022024-02-13T21:46:55.855Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533348365Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850,"session_id":"26e384d3-c0f4-47ad-8796-95229d1e2cc9"}
192032024-02-13T21:46:55.855Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:54.533364569Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
192042024-02-13T21:46:55.858ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
192052024-02-13T21:46:55.858ZLoading write count information from file "/tmp/test_repair_verify.data"
192062024-02-13T21:46:55.858ZRun Repair workload
192072024-02-13T21:46:55.858Z01/30 Write block 142 len 24576 data: 20 17 19 17 13 13
192082024-02-13T21:46:55.858Z02/30 Flush
192092024-02-13T21:46:55.858Z03/30 Write block 279 len 4096 data: 18
192102024-02-13T21:46:55.858Z04/30 Write block 594 len 4096 data: 8
192112024-02-13T21:46:55.858Z05/30 Read block 134 len 16384
192122024-02-13T21:46:55.858Z06/30 Read block 256 len 8192
192132024-02-13T21:46:55.858Z07/30 Write block 572 len 40960 data: 7 5 6 6 8 9 8 6 7 8
192142024-02-13T21:46:55.861Z08/30 Read block 343 len 36864
192152024-02-13T21:46:55.861Z09/30 Read block 151 len 20480
192162024-02-13T21:46:55.862Z10/30 Read block 214 len 40960
192172024-02-13T21:46:55.864Z11/30 Write block 132 len 12288 data: 15 15 14
192182024-02-13T21:46:55.864Z12/30 Write block 404 len 4096 data: 15
192192024-02-13T21:46:55.865Z13/30 Write block 568 len 24576 data: 13 11 9 7 8 6
192202024-02-13T21:46:55.865Z14/30 Read block 147 len 8192
192212024-02-13T21:46:55.868Z15/30 Write block 253 len 24576 data: 12 11 13 15 16 13
192222024-02-13T21:46:55.868Z16/30 Read block 285 len 36864
192232024-02-13T21:46:55.868Z17/30 Read block 552 len 8192
192242024-02-13T21:46:55.868Z18/30 Write block 62 len 24576 data: 17 16 11 11 12 14
192252024-02-13T21:46:55.871Z19/30 Read block 579 len 28672
192262024-02-13T21:46:55.871Z20/30 Write block 54 len 40960 data: 13 15 16 16 17 21 18 19 18 17
192272024-02-13T21:46:55.871Z21/30 Read block 151 len 12288
192282024-02-13T21:46:55.874Z22/30 Read block 557 len 24576
192292024-02-13T21:46:55.874Z23/30 Flush
192302024-02-13T21:46:55.877Z24/30 Write block 561 len 24576 data: 11 11 11 13 13 12
192312024-02-13T21:46:55.877Z25/30 Flush
192322024-02-13T21:46:55.877Z26/30 Write block 48 len 4096 data: 12
192332024-02-13T21:46:55.883Z27/30 Write block 30 len 24576 data: 12 14 15 15 17 19
192342024-02-13T21:46:55.883Z28/30 Write block 4 len 24576 data: 8 11 11 11 10 12
192352024-02-13T21:46:55.883Z29/30 Write block 161 len 28672 data: 14 14 12 10 8 11 8
192362024-02-13T21:46:55.883Z30/30 Write block 371 len 28672 data: 18 21 19 15 16 15 13
192372024-02-13T21:46:55.887Z----------------------------------------------------------------
192382024-02-13T21:46:55.887Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:28
192392024-02-13T21:46:55.887ZStates: Active Active Active
192402024-02-13T21:46:55.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192412024-02-13T21:46:55.887Z 3 Acked 1002 Write 1 Sent Done Done false
192422024-02-13T21:46:55.887Z 4 Acked 1003 Write 1 Sent Done Done false
192432024-02-13T21:46:55.887Z 5 Acked 1004 Read 4 Sent Done Done false
192442024-02-13T21:46:55.887Z 6 Acked 1005 Read 2 Sent Done Done false
192452024-02-13T21:46:55.887Z 7 Acked 1006 Write 10 Sent Done Done false
192462024-02-13T21:46:55.887Z 8 Acked 1007 Read 9 Sent Done Done false
192472024-02-13T21:46:55.887Z 9 Acked 1008 Read 5 Sent Done Done false
192482024-02-13T21:46:55.887Z 10 Acked 1009 Read 10 Sent Done Done false
192492024-02-13T21:46:55.887Z 11 Acked 1010 Write 3 Sent Done Done false
192502024-02-13T21:46:55.887Z 12 Acked 1011 Write 1 Sent Done Done false
192512024-02-13T21:46:55.887Z 13 Acked 1012 Write 6 Sent Done Done false
192522024-02-13T21:46:55.887Z 14 Acked 1013 Read 2 Sent Done Done false
192532024-02-13T21:46:55.887Z 15 Acked 1014 Write 6 Sent Done Done false
192542024-02-13T21:46:55.887Z 16 Acked 1015 Read 9 Sent Done Done false
192552024-02-13T21:46:55.888Z 17 Acked 1016 Read 2 Sent Done Done false
192562024-02-13T21:46:55.888Z 18 Acked 1017 Write 6 Sent Done Done false
192572024-02-13T21:46:55.888Z 19 Acked 1018 Read 7 Sent Done Done false
192582024-02-13T21:46:55.888Z 20 Acked 1019 Write 10 Sent Done Done false
192592024-02-13T21:46:55.888Z 21 Acked 1020 Read 3 Sent Done Done false
192602024-02-13T21:46:55.888Z 22 Acked 1021 Read 6 Sent Done Done false
192612024-02-13T21:46:55.888Z 23 Acked 1022 Flush 0 Sent Done Done false
192622024-02-13T21:46:55.888Z 24 Acked 1023 Write 6 Sent Done Done false
192632024-02-13T21:46:55.888Z 25 Acked 1024 Flush 0 Sent Done Done false
192642024-02-13T21:46:55.888Z 26 Acked 1025 Write 1 Sent Done Done false
192652024-02-13T21:46:55.888Z 27 Acked 1026 Write 6 Sent Done Done false
192662024-02-13T21:46:55.888Z 28 Acked 1027 Write 6 Sent Done Done false
192672024-02-13T21:46:55.888Z 29 Acked 1028 Write 7 Sent Sent Done false
192682024-02-13T21:46:55.888Z 30 Acked 1029 Write 7 Sent Sent Sent false
192692024-02-13T21:46:55.888Z STATES DS:0 DS:1 DS:2 TOTAL
192702024-02-13T21:46:55.888Z New 0 0 0 0
192712024-02-13T21:46:55.888Z Sent 28 2 1 31
192722024-02-13T21:46:55.888Z Done 0 26 27 53
192732024-02-13T21:46:55.888Z Skipped 0 0 0 0
192742024-02-13T21:46:55.888Z Error 0 0 0 0
192752024-02-13T21:46:55.888ZLast Flush: 1001 1024 1024
192762024-02-13T21:46:55.888ZDownstairs last five completed: 1001 1000
192772024-02-13T21:46:55.888ZUpstairs last five completed: 30 29 28 27 26
192782024-02-13T21:46:55.888Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:54.563703203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1850}
192792024-02-13T21:46:55.888ZWrote out file "/tmp/test_repair_verify.data"
192802024-02-13T21:46:55.889Z
192812024-02-13T21:46:55.892ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192822024-02-13T21:46:55.904ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192832024-02-13T21:46:55.904Z 0 000-019 172 172 172 314 314 314 F T T <---
192842024-02-13T21:46:55.904Z 1 020-039 176 176 176 319 319 319 F T T <---
192852024-02-13T21:46:55.904Z 2 040-059 173 178 178 315 326 326 F T T <---
192862024-02-13T21:46:55.905Z 3 060-079 176 178 178 323 326 326 F F F <---
192872024-02-13T21:46:55.908Z 4 080-099 177 177 177 324 324 324 F F F
192882024-02-13T21:46:55.908Z 5 100-119 172 172 172 314 314 314 F F F
192892024-02-13T21:46:55.908Z 6 120-139 151 178 178 284 326 326 F F F <---
192902024-02-13T21:46:55.908Z 7 140-159 178 178 178 325 325 325 F F F
192912024-02-13T21:46:55.908Z 8 160-179 173 173 173 315 315 315 F T T <---
192922024-02-13T21:46:55.908Z 9 180-199 177 177 177 324 324 324 F F F
192932024-02-13T21:46:55.908Z 10 200-219 176 176 176 321 321 321 F F F
192942024-02-13T21:46:55.908Z 11 220-239 173 173 173 315 315 315 F F F
192952024-02-13T21:46:55.908Z 12 240-259 173 178 178 315 326 326 F F F <---
192962024-02-13T21:46:55.908Z 13 260-279 163 178 178 303 326 326 F F F <---
192972024-02-13T21:46:55.908Z 14 280-299 176 176 176 321 321 321 F F F
192982024-02-13T21:46:55.908Z 15 300-319 158 158 158 297 297 297 F F F
192992024-02-13T21:46:55.908Z 16 320-339 176 176 176 322 322 322 F F F
193002024-02-13T21:46:55.909Z 17 340-359 174 174 174 317 317 317 F F F
193012024-02-13T21:46:55.909Z 18 360-379 172 172 172 314 314 314 F T T <---
193022024-02-13T21:46:55.909Z 19 380-399 176 176 176 321 321 321 F F F
193032024-02-13T21:46:55.909Z 20 400-419 175 178 178 318 326 326 F F F <---
193042024-02-13T21:46:55.909Z 21 420-439 174 174 174 317 317 317 F F F
193052024-02-13T21:46:55.909Z 22 440-459 170 170 170 312 312 312 F F F
193062024-02-13T21:46:55.909Z 23 460-479 173 173 173 315 315 315 F F F
193072024-02-13T21:46:55.909Z 24 480-499 177 177 177 324 324 324 F F F
193082024-02-13T21:46:55.909Z 25 500-519 175 175 175 318 318 318 F F F
193092024-02-13T21:46:55.909Z 26 520-539 176 176 176 321 321 321 F F F
193102024-02-13T21:46:55.909Z 27 540-559 175 175 175 318 318 318 F F F
193112024-02-13T21:46:55.909Z 28 560-579 171 178 178 313 327 327 F F F <---
193122024-02-13T21:46:55.909Z 29 580-599 174 178 178 316 326 326 F F F <---
193132024-02-13T21:46:55.909ZMax gen: 178, Max flush: 327
193142024-02-13T21:46:55.909ZError: Difference in extent metadata found!
193152024-02-13T21:46:55.909ZA Difference in extent metadata is expected here
193162024-02-13T21:46:55.909ZOn loop 89
193172024-02-13T21:46:55.909Z
193182024-02-13T21:46:55.912ZVerifying data now
193192024-02-13T21:47:00.496ZNo change after verify
193202024-02-13T21:47:00.496ZLoop: 89 Downstairs dump after verify (and repair):
193212024-02-13T21:47:00.510ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193222024-02-13T21:47:00.510Z 0 000-019 179 179 179 328 328 328 F F F
193232024-02-13T21:47:00.510Z 1 020-039 179 179 179 328 328 328 F F F
193242024-02-13T21:47:00.510Z 2 040-059 179 179 179 328 328 328 F F F
193252024-02-13T21:47:00.510Z 3 060-079 178 178 178 326 326 326 F F F
193262024-02-13T21:47:00.510Z 4 080-099 177 177 177 324 324 324 F F F
193272024-02-13T21:47:00.511Z 5 100-119 172 172 172 314 314 314 F F F
193282024-02-13T21:47:00.511Z 6 120-139 178 178 178 326 326 326 F F F
193292024-02-13T21:47:00.517Z 7 140-159 178 178 178 325 325 325 F F F
193302024-02-13T21:47:00.517Z 8 160-179 179 179 179 328 328 328 F F F
193312024-02-13T21:47:00.517Z 9 180-199 177 177 177 324 324 324 F F F
193322024-02-13T21:47:00.517Z 10 200-219 176 176 176 321 321 321 F F F
193332024-02-13T21:47:00.517Z 11 220-239 173 173 173 315 315 315 F F F
193342024-02-13T21:47:00.517Z 12 240-259 178 178 178 326 326 326 F F F
193352024-02-13T21:47:00.517Z 13 260-279 178 178 178 326 326 326 F F F
193362024-02-13T21:47:00.517Z 14 280-299 176 176 176 321 321 321 F F F
193372024-02-13T21:47:00.517Z 15 300-319 158 158 158 297 297 297 F F F
193382024-02-13T21:47:00.517Z 16 320-339 176 176 176 322 322 322 F F F
193392024-02-13T21:47:00.517Z 17 340-359 174 174 174 317 317 317 F F F
193402024-02-13T21:47:00.517Z 18 360-379 179 179 179 328 328 328 F F F
193412024-02-13T21:47:00.517Z 19 380-399 176 176 176 321 321 321 F F F
193422024-02-13T21:47:00.517Z 20 400-419 178 178 178 326 326 326 F F F
193432024-02-13T21:47:00.517Z 21 420-439 174 174 174 317 317 317 F F F
193442024-02-13T21:47:00.517Z 22 440-459 170 170 170 312 312 312 F F F
193452024-02-13T21:47:00.517Z 23 460-479 173 173 173 315 315 315 F F F
193462024-02-13T21:47:00.517Z 24 480-499 177 177 177 324 324 324 F F F
193472024-02-13T21:47:00.517Z 25 500-519 175 175 175 318 318 318 F F F
193482024-02-13T21:47:00.517Z 26 520-539 176 176 176 321 321 321 F F F
193492024-02-13T21:47:00.517Z 27 540-559 175 175 175 318 318 318 F F F
193502024-02-13T21:47:00.517Z 28 560-579 178 178 178 327 327 327 F F F
193512024-02-13T21:47:00.517Z 29 580-599 178 178 178 326 326 326 F F F
193522024-02-13T21:47:00.518ZMax gen: 179, Max flush: 328
193532024-02-13T21:47:00.518Z
193542024-02-13T21:47:00.518ZBegin loop 90
193552024-02-13T21:47:00.518ZDownstairs to restart: 1
193562024-02-13T21:47:00.531Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.211646607Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
193572024-02-13T21:47:00.531Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
193582024-02-13T21:47:00.531Z,"time":"2024-02-13T21:46:59.21196781Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
193592024-02-13T21:47:00.536Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212071133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
193602024-02-13T21:47:00.536Z{"msg":"Crucible aaa740c0-2f7a-49e3-adec-88a19375a672 has session id: 84960b3d-4b52-4909-9796-fbdcf4989bdc","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212124713Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193612024-02-13T21:47:00.536Z{"msg":"Upstairs opts: Upstairs UUID: aaa740c0-2f7a-49e3-adec-88a19375a672, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212182532Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193622024-02-13T21:47:00.536Z{"msg":"Crucible stats registered with UUID: aaa740c0-2f7a-49e3-adec-88a19375a672","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212214441Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193632024-02-13T21:47:00.536Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212260824Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
193642024-02-13T21:47:00.536Z{"msg":"aaa740c0-2f7a-49e3-adec-88a19375a672 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21229821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193652024-02-13T21:47:00.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212324161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193662024-02-13T21:47:00.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212364626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193672024-02-13T21:47:00.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212397504Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193682024-02-13T21:47:00.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212457743Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193692024-02-13T21:47:00.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212521489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193702024-02-13T21:47:00.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212606509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193712024-02-13T21:47:00.537Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212639726Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193722024-02-13T21:47:00.537Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212669316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193732024-02-13T21:47:00.537Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212700065Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193742024-02-13T21:47:00.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212725905Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193752024-02-13T21:47:00.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212763392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193762024-02-13T21:47:00.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.212788173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"io task","client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193772024-02-13T21:47:00.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213272727Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193782024-02-13T21:47:00.537Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213307744Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193792024-02-13T21:47:00.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213323998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193802024-02-13T21:47:00.537Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213356426Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193812024-02-13T21:47:00.537Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213829245Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193822024-02-13T21:47:00.537Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.213959398Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193832024-02-13T21:47:00.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21414737Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193842024-02-13T21:47:00.537Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214200131Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193852024-02-13T21:47:00.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214378806Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193862024-02-13T21:47:00.537Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21442251Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193872024-02-13T21:47:00.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214461786Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193882024-02-13T21:47:00.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214497523Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193892024-02-13T21:47:00.537Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214513357Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193902024-02-13T21:47:00.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214540377Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193912024-02-13T21:47:00.537Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.214714944Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193922024-02-13T21:47:00.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215105942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193932024-02-13T21:47:00.538Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21514003Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193942024-02-13T21:47:00.538Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 328, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.2151662Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193952024-02-13T21:47:00.538Z{"msg":"[0]R generation[0..12]: [179, 179, 179, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21519499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193962024-02-13T21:47:00.538Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21522063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193972024-02-13T21:47:00.538Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 328, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21524927Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193982024-02-13T21:47:00.538Z{"msg":"[1]R generation[0..12]: [179, 179, 179, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215274361Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
193992024-02-13T21:47:00.538Z{"msg":"Wait for a query_work_queue command to finish before sending IO
194002024-02-13T21:47:00.538Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21530298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194012024-02-13T21:47:00.544Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 328, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215336048Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194022024-02-13T21:47:00.544Z{"msg":"[2]R generation[0..12]: [179, 179, 179, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215361349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194032024-02-13T21:47:00.544Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215393008Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194042024-02-13T21:47:00.544Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215417719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194052024-02-13T21:47:00.544Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215447298Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194062024-02-13T21:47:00.544Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21547041Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194072024-02-13T21:47:00.544Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.2154969Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"":"downstairs","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194082024-02-13T21:47:00.544Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215520162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194092024-02-13T21:47:00.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215545323Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"0","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194102024-02-13T21:47:00.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215568254Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"1","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194112024-02-13T21:47:00.544Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215594375Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"client":"2","":"downstairs","session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194122024-02-13T21:47:00.544Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215617516Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194132024-02-13T21:47:00.544Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215643177Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194142024-02-13T21:47:00.544Z{"msg":"aaa740c0-2f7a-49e3-adec-88a19375a672 is now active with session: 84960b3d-4b52-4909-9796-fbdcf4989bdc","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215666299Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194152024-02-13T21:47:00.545Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.21569087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858,"session_id":"84960b3d-4b52-4909-9796-fbdcf4989bdc"}
194162024-02-13T21:47:00.545Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:46:59.215712432Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
194172024-02-13T21:47:00.545ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194182024-02-13T21:47:00.545ZLoading write count information from file "/tmp/test_repair_verify.data"
194192024-02-13T21:47:00.545ZRun Repair workload
194202024-02-13T21:47:00.545Z01/30 Write block 44 len 12288 data: 9 9 11
194212024-02-13T21:47:00.545Z02/30 Read block 562 len 24576
194222024-02-13T21:47:00.545Z03/30 Read block 190 len 16384
194232024-02-13T21:47:00.545Z04/30 Read block 60 len 8192
194242024-02-13T21:47:00.545Z05/30 Read block 496 len 4096
194252024-02-13T21:47:00.545Z06/30 Write block 247 len 32768 data: 9 9 13 15 17 14 13 12
194262024-02-13T21:47:00.545Z07/30 Flush
194272024-02-13T21:47:00.548Z08/30 Write block 391 len 20480 data: 10 10 12 13 11
194282024-02-13T21:47:00.548Z09/30 Read block 62 len 12288
194292024-02-13T21:47:00.549Z10/30 Flush
194302024-02-13T21:47:00.549Z11/30 Flush
194312024-02-13T21:47:00.549Z12/30 Write block 351 len 40960 data: 12 14 15 16 17 17 16 15 13 13
194322024-02-13T21:47:00.549Z13/30 Read block 101 len 8192
194332024-02-13T21:47:00.552Z14/30 Read block 142 len 20480
194342024-02-13T21:47:00.552Z15/30 Read block 212 len 32768
194352024-02-13T21:47:00.552Z16/30 Read block 79 len 16384
194362024-02-13T21:47:00.555Z17/30 Write block 495 len 4096 data: 15
194372024-02-13T21:47:00.555Z18/30 Read block 206 len 40960
194382024-02-13T21:47:00.555Z19/30 Read block 565 len 16384
194392024-02-13T21:47:00.555Z20/30 Read block 101 len 36864
194402024-02-13T21:47:00.558Z21/30 Write block 282 len 12288 data: 8 8 10
194412024-02-13T21:47:00.558Z22/30 Read block 566 len 24576
194422024-02-13T21:47:00.558Z23/30 Read block 32 len 20480
194432024-02-13T21:47:00.558Z24/30 Write block 545 len 4096 data: 15
194442024-02-13T21:47:00.561Z25/30 Read block 562 len 40960
194452024-02-13T21:47:00.561Z26/30 Write block 404 len 32768 data: 16 14 12 13 16 18 12 17
194462024-02-13T21:47:00.561Z27/30 Read block 148 len 24576
194472024-02-13T21:47:00.565Z28/30 Read block 475 len 12288
194482024-02-13T21:47:00.565Z29/30 Read block 188 len 20480
194492024-02-13T21:47:00.565Z30/30 Write block 441 len 28672 data: 12 12 11 10 15 14 16
194502024-02-13T21:47:00.565Z----------------------------------------------------------------
194512024-02-13T21:47:00.565Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194522024-02-13T21:47:00.565ZStates: Active Active Active
194532024-02-13T21:47:00.566ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194542024-02-13T21:47:00.566Z 1 Acked 1000 Write 3 Done Done Done false
194552024-02-13T21:47:00.566Z 2 Acked 1001 Read 6 Done Done Done false
194562024-02-13T21:47:00.566Z 3 Acked 1002 Read 4 Done Sent Done false
194572024-02-13T21:47:00.566Z 4 Acked 1003 Read 2 Done Sent Done false
194582024-02-13T21:47:00.566Z 5 Acked 1004 Read 1 Done Sent Done false
194592024-02-13T21:47:00.566Z 6 Acked 1005 Write 8 Done Sent Done false
194602024-02-13T21:47:00.566Z 7 Acked 1006 Flush 0 Done Sent Done false
194612024-02-13T21:47:00.566Z 8 Acked 1007 Write 5 Done Sent Done false
194622024-02-13T21:47:00.566Z 9 Acked 1008 Read 3 Done Sent Done false
194632024-02-13T21:47:00.566Z 10 Acked 1009 Flush 0 Done Sent Done false
194642024-02-13T21:47:00.566Z 11 Acked 1010 Flush 0 Done Sent Done false
194652024-02-13T21:47:00.566Z 12 Acked 1011 Write 10 Done Sent Done false
194662024-02-13T21:47:00.566Z 13 Acked 1012 Read 2 Done Sent Done false
194672024-02-13T21:47:00.566Z 14 Acked 1013 Read 5 Done Sent Done false
194682024-02-13T21:47:00.566Z 15 Acked 1014 Read 8 Done Sent Done false
194692024-02-13T21:47:00.566Z 16 Acked 1015 Read 4 Done Sent Done false
194702024-02-13T21:47:00.566Z 17 Acked 1016 Write 1 Done Sent Done false
194712024-02-13T21:47:00.566Z 18 Acked 1017 Read 10 Done Sent Done false
194722024-02-13T21:47:00.566Z 19 Acked 1018 Read 4 Done Sent Done false
194732024-02-13T21:47:00.566Z 20 Acked 1019 Read 9 Done Sent Done false
194742024-02-13T21:47:00.566Z 21 Acked 1020 Write 3 Done Sent Done false
194752024-02-13T21:47:00.566Z 22 Acked 1021 Read 6 Done Sent Done false
194762024-02-13T21:47:00.566Z 23 Acked 1022 Read 5 Done Sent Done false
194772024-02-13T21:47:00.567Z 24 Acked 1023 Write 1 Done Sent Done false
194782024-02-13T21:47:00.567Z 25 Acked 1024 Read 10 Done Sent Done false
194792024-02-13T21:47:00.567Z 26 Acked 1025 Write 8 Done Sent Done false
194802024-02-13T21:47:00.567Z 27 Acked 1026 Read 6 Done Sent Done false
194812024-02-13T21:47:00.567Z 28 Acked 1027 Read 3 Done Sent Done false
194822024-02-13T21:47:00.567Z 29 Acked 1028 Read 5 Done Sent Done false
194832024-02-13T21:47:00.567Z 30 Acked 1029 Write 7 Sent Sent Sent false
194842024-02-13T21:47:00.567Z STATES DS:0 DS:1 DS:2 TOTAL
194852024-02-13T21:47:00.567Z New 0 0 0 0
194862024-02-13T21:47:00.567Z Sent 1 28 1 30
194872024-02-13T21:47:00.567Z Done 29 2 29 60
194882024-02-13T21:47:00.567Z Skipped 0 0 0 0
194892024-02-13T21:47:00.567Z Error 0 0 0 0
194902024-02-13T21:47:00.567ZLast Flush: 1010 0 1010
194912024-02-13T21:47:00.567ZDownstairs last five completed:
194922024-02-13T21:47:00.567ZUpstairs last five completed: 30 29 28 27 26
194932024-02-13T21:47:00.567Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:46:59.24507959Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1858}
194942024-02-13T21:47:00.567ZWrote out file "/tmp/test_repair_verify.data"
194952024-02-13T21:47:00.571Z
194962024-02-13T21:47:00.575ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194972024-02-13T21:47:00.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194982024-02-13T21:47:00.588Z 0 000-019 179 179 179 328 328 328 F F F
194992024-02-13T21:47:00.589Z 1 020-039 179 179 179 328 328 328 F F F
195002024-02-13T21:47:00.589Z 2 040-059 180 179 180 329 328 329 F T F <---
195012024-02-13T21:47:00.589Z 3 060-079 178 178 178 326 326 326 F F F
195022024-02-13T21:47:00.589Z 4 080-099 177 177 177 324 324 324 F F F
195032024-02-13T21:47:00.593Z 5 100-119 172 172 172 314 314 314 F F F
195042024-02-13T21:47:00.593Z 6 120-139 178 178 178 326 326 326 F F F
195052024-02-13T21:47:00.593Z 7 140-159 178 178 178 325 325 325 F F F
195062024-02-13T21:47:00.593Z 8 160-179 179 179 179 328 328 328 F F F
195072024-02-13T21:47:00.593Z 9 180-199 177 177 177 324 324 324 F F F
195082024-02-13T21:47:00.593Z 10 200-219 176 176 176 321 321 321 F F F
195092024-02-13T21:47:00.593Z 11 220-239 173 173 173 315 315 315 F F F
195102024-02-13T21:47:00.593Z 12 240-259 180 178 180 329 326 329 F F F <---
195112024-02-13T21:47:00.593Z 13 260-279 178 178 178 326 326 326 F F F
195122024-02-13T21:47:00.593Z 14 280-299 176 176 176 321 321 321 T F T <---
195132024-02-13T21:47:00.593Z 15 300-319 158 158 158 297 297 297 F F F
195142024-02-13T21:47:00.593Z 16 320-339 176 176 176 322 322 322 F F F
195152024-02-13T21:47:00.593Z 17 340-359 174 174 174 317 317 317 T F T <---
195162024-02-13T21:47:00.593Z 18 360-379 179 179 179 328 328 328 T F T <---
195172024-02-13T21:47:00.594Z 19 380-399 180 176 180 330 321 330 F F F <---
195182024-02-13T21:47:00.594Z 20 400-419 178 178 178 326 326 326 T F T <---
195192024-02-13T21:47:00.594Z 21 420-439 174 174 174 317 317 317 F F F
195202024-02-13T21:47:00.594Z 22 440-459 170 170 170 312 312 312 T F T <---
195212024-02-13T21:47:00.594Z 23 460-479 173 173 173 315 315 315 F F F
195222024-02-13T21:47:00.594Z 24 480-499 177 177 177 324 324 324 T F T <---
195232024-02-13T21:47:00.594Z 25 500-519 175 175 175 318 318 318 F F F
195242024-02-13T21:47:00.594Z 26 520-539 176 176 176 321 321 321 F F F
195252024-02-13T21:47:00.594Z 27 540-559 175 175 175 318 318 318 T F T <---
195262024-02-13T21:47:00.594Z 28 560-579 178 178 178 327 327 327 F F F
195272024-02-13T21:47:00.594Z 29 580-599 178 178 178 326 326 326 F F F
195282024-02-13T21:47:00.594ZMax gen: 180, Max flush: 330
195292024-02-13T21:47:00.594ZError: Difference in extent metadata found!
195302024-02-13T21:47:00.594ZA Difference in extent metadata is expected here
195312024-02-13T21:47:00.594ZOn loop 90
195322024-02-13T21:47:00.594Z
195332024-02-13T21:47:00.597ZVerifying data now
195342024-02-13T21:47:05.119ZNo change after verify
195352024-02-13T21:47:05.119ZLoop: 90 Downstairs dump after verify (and repair):
195362024-02-13T21:47:05.132ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195372024-02-13T21:47:05.132Z 0 000-019 179 179 179 328 328 328 F F F
195382024-02-13T21:47:05.132Z 1 020-039 179 179 179 328 328 328 F F F
195392024-02-13T21:47:05.132Z 2 040-059 180 180 180 329 329 329 F F F
195402024-02-13T21:47:05.132Z 3 060-079 178 178 178 326 326 326 F F F
195412024-02-13T21:47:05.132Z 4 080-099 177 177 177 324 324 324 F F F
195422024-02-13T21:47:05.137Z 5 100-119 172 172 172 314 314 314 F F F
195432024-02-13T21:47:05.137Z 6 120-139 178 178 178 326 326 326 F F F
195442024-02-13T21:47:05.137Z 7 140-159 178 178 178 325 325 325 F F F
195452024-02-13T21:47:05.137Z 8 160-179 179 179 179 328 328 328 F F F
195462024-02-13T21:47:05.137Z 9 180-199 177 177 177 324 324 324 F F F
195472024-02-13T21:47:05.137Z 10 200-219 176 176 176 321 321 321 F F F
195482024-02-13T21:47:05.137Z 11 220-239 173 173 173 315 315 315 F F F
195492024-02-13T21:47:05.137Z 12 240-259 180 180 180 329 329 329 F F F
195502024-02-13T21:47:05.137Z 13 260-279 178 178 178 326 326 326 F F F
195512024-02-13T21:47:05.137Z 14 280-299 181 181 181 331 331 331 F F F
195522024-02-13T21:47:05.138Z 15 300-319 158 158 158 297 297 297 F F F
195532024-02-13T21:47:05.138Z 16 320-339 176 176 176 322 322 322 F F F
195542024-02-13T21:47:05.138Z 17 340-359 181 181 181 331 331 331 F F F
195552024-02-13T21:47:05.138Z 18 360-379 181 181 181 331 331 331 F F F
195562024-02-13T21:47:05.138Z 19 380-399 180 180 180 330 330 330 F F F
195572024-02-13T21:47:05.138Z 20 400-419 181 181 181 331 331 331 F F F
195582024-02-13T21:47:05.138Z 21 420-439 174 174 174 317 317 317 F F F
195592024-02-13T21:47:05.138Z 22 440-459 181 181 181 331 331 331 F F F
195602024-02-13T21:47:05.138Z 23 460-479 173 173 173 315 315 315 F F F
195612024-02-13T21:47:05.138Z 24 480-499 181 181 181 331 331 331 F F F
195622024-02-13T21:47:05.138Z 25 500-519 175 175 175 318 318 318 F F F
195632024-02-13T21:47:05.138Z 26 520-539 176 176 176 321 321 321 F F F
195642024-02-13T21:47:05.138Z 27 540-559 181 181 181 331 331 331 F F F
195652024-02-13T21:47:05.139Z 28 560-579 178 178 178 327 327 327 F F F
195662024-02-13T21:47:05.139Z 29 580-599 178 178 178 326 326 326 F F F
195672024-02-13T21:47:05.139ZMax gen: 181, Max flush: 331
195682024-02-13T21:47:05.139Z
195692024-02-13T21:47:05.139ZBegin loop 91
195702024-02-13T21:47:05.139ZDownstairs to restart: 2
195712024-02-13T21:47:05.153Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831573142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
195722024-02-13T21:47:05.153Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
195732024-02-13T21:47:05.159Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831857878Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
195742024-02-13T21:47:05.159Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831894665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
195752024-02-13T21:47:05.160Z{"msg":"Crucible 712f0fdb-6ca0-4191-a2a8-e77f63a4bf52 has session id: a5c19861-8369-4a9f-99ac-ee3484bf5a66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831916317Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195762024-02-13T21:47:05.160Z{"msg":"Upstairs opts: Upstairs UUID: 712f0fdb-6ca0-4191-a2a8-e77f63a4bf52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831960041Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195772024-02-13T21:47:05.160Z{"msg":"Crucible stats registered with UUID: 712f0fdb-6ca0-4191-a2a8-e77f63a4bf52","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.831984952Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195782024-02-13T21:47:05.160Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.83201773Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
195792024-02-13T21:47:05.160Z{"msg":"712f0fdb-6ca0-4191-a2a8-e77f63a4bf52 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832059195Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195802024-02-13T21:47:05.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832121383Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195812024-02-13T21:47:05.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832173104Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195822024-02-13T21:47:05.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832199794Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195832024-02-13T21:47:05.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832242539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195842024-02-13T21:47:05.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832289472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195852024-02-13T21:47:05.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832330947Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195862024-02-13T21:47:05.160Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832368363Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195872024-02-13T21:47:05.160Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832418025Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195882024-02-13T21:47:05.160Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832456831Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195892024-02-13T21:47:05.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.83248606Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195902024-02-13T21:47:05.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832522307Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195912024-02-13T21:47:05.160Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.832592902Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"io task","client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195922024-02-13T21:47:05.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.833268537Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195932024-02-13T21:47:05.160Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.833306353Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195942024-02-13T21:47:05.160Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.833323137Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195952024-02-13T21:47:05.160Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.833392152Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195962024-02-13T21:47:05.160Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.833884953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195972024-02-13T21:47:05.160Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834065367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195982024-02-13T21:47:05.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834392999Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
195992024-02-13T21:47:05.161Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834438692Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196002024-02-13T21:47:05.161Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834456796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196012024-02-13T21:47:05.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834538186Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196022024-02-13T21:47:05.161Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834565266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196032024-02-13T21:47:05.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834580071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196042024-02-13T21:47:05.161Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834594076Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196052024-02-13T21:47:05.161Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.834652075Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196062024-02-13T21:47:05.161Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835103671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196072024-02-13T21:47:05.161Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835506675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196082024-02-13T21:47:05.161Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835544381Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196092024-02-13T21:47:05.161Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 329, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835561865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196102024-02-13T21:47:05.162Z{"msg":"[0]R generation[0..12]: [179, 179, 180, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835594163Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196112024-02-13T21:47:05.162Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835625312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196122024-02-13T21:47:05.162Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 329, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.8356588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196132024-02-13T21:47:05.162ZWait for a query_work_queue command to finish before sending IO
196142024-02-13T21:47:05.162Z{"msg":"[1]R generation[0..12]: [179, 179, 180, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835693287Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196152024-02-13T21:47:05.162Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835726105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196162024-02-13T21:47:05.162Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 329, 326, 324, 314, 326, 325, 328, 324, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835759193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196172024-02-13T21:47:05.162Z{"msg":"[2]R generation[0..12]: [179, 179, 180, 178, 177, 172, 178, 178, 179, 177, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835791221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196182024-02-13T21:47:05.162Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.8358243Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196192024-02-13T21:47:05.162Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835855978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196202024-02-13T21:47:05.162Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835886087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196212024-02-13T21:47:05.162Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835918095Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196222024-02-13T21:47:05.162Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835948055Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"":"downstairs","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196232024-02-13T21:47:05.162Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.835980343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196242024-02-13T21:47:05.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836008873Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"0","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196252024-02-13T21:47:05.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836027856Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"1","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196262024-02-13T21:47:05.163Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836056675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"client":"2","":"downstairs","session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196272024-02-13T21:47:05.163ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196282024-02-13T21:47:05.163Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836080676Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196292024-02-13T21:47:05.163Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836107267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196302024-02-13T21:47:05.163Z{"msg":"712f0fdb-6ca0-4191-a2a8-e77f63a4bf52 is now active with session: a5c19861-8369-4a9f-99ac-ee3484bf5a66","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.83612557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196312024-02-13T21:47:05.163Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836151761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866,"session_id":"a5c19861-8369-4a9f-99ac-ee3484bf5a66"}
196322024-02-13T21:47:05.163Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:03.836169584Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
196332024-02-13T21:47:05.163ZLoading write count information from file "/tmp/test_repair_verify.data"
196342024-02-13T21:47:05.163ZRun Repair workload
196352024-02-13T21:47:05.170Z01/30 Write block 261 len 20480 data: 8 9 10 11 10
196362024-02-13T21:47:05.170Z02/30 Read block 74 len 4096
196372024-02-13T21:47:05.170Z03/30 Write block 88 len 36864 data: 13 13 13 13 12 11 10 12 13
196382024-02-13T21:47:05.170Z04/30 Write block 375 len 4096 data: 17
196392024-02-13T21:47:05.170Z05/30 Write block 405 len 8192 data: 15 13
196402024-02-13T21:47:05.170Z06/30 Write block 579 len 8192 data: 7 8
196412024-02-13T21:47:05.170Z07/30 Read block 251 len 32768
196422024-02-13T21:47:05.170Z08/30 Read block 111 len 40960
196432024-02-13T21:47:05.170Z09/30 Read block 422 len 8192
196442024-02-13T21:47:05.170Z10/30 Write block 249 len 20480 data: 14 16 18 15 14
196452024-02-13T21:47:05.170Z11/30 Write block 40 len 36864 data: 14 13 11 11 10 10 12 10 13
196462024-02-13T21:47:05.170Z12/30 Write block 459 len 12288 data: 18 20 19
196472024-02-13T21:47:05.170Z13/30 Read block 101 len 32768
196482024-02-13T21:47:05.170Z14/30 Read block 427 len 36864
196492024-02-13T21:47:05.174Z15/30 Write block 447 len 40960 data: 17 14 17 17 15 13 15 15 15 18
196502024-02-13T21:47:05.174Z16/30 Write block 447 len 32768 data: 18 15 18 18 16 14 16 16
196512024-02-13T21:47:05.174Z17/30 Read block 158 len 28672
196522024-02-13T21:47:05.174Z18/30 Write block 91 len 32768 data: 14 13 12 11 13 14 13 12
196532024-02-13T21:47:05.174Z19/30 Write block 47 len 16384 data: 11 14 12 12
196542024-02-13T21:47:05.178Z20/30 Read block 139 len 36864
196552024-02-13T21:47:05.178Z21/30 Write block 165 len 40960 data: 9 12 9 7 7 5 7 11 8 8
196562024-02-13T21:47:05.178Z22/30 Flush
196572024-02-13T21:47:05.183Z23/30 Write block 282 len 36864 data: 9 9 11 11 10 10 11 12 11
196582024-02-13T21:47:05.188Z24/30 Read block 294 len 40960
196592024-02-13T21:47:05.188Z25/30 Read block 77 len 4096
196602024-02-13T21:47:05.188Z26/30 Read block 223 len 24576
196612024-02-13T21:47:05.188Z27/30 Write block 403 len 16384 data: 15 17 16 14
196622024-02-13T21:47:05.188Z28/30 Read block 591 len 8192
196632024-02-13T21:47:05.188Z29/30 Write block 543 len 24576 data: 19 14 16 14 12 11
196642024-02-13T21:47:05.193Z30/30 Write block 193 len 8192 data: 12 15
196652024-02-13T21:47:05.193Z----------------------------------------------------------------
196662024-02-13T21:47:05.193Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196672024-02-13T21:47:05.193ZStates: Active Active Active
196682024-02-13T21:47:05.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196692024-02-13T21:47:05.193Z 1 Acked 1000 Write 5 Done Done Sent false
196702024-02-13T21:47:05.194Z 2 Acked 1001 Read 1 Done Done Sent false
196712024-02-13T21:47:05.194Z 3 Acked 1002 Write 9 Done Done Sent false
196722024-02-13T21:47:05.194Z 4 Acked 1003 Write 1 Done Done Sent false
196732024-02-13T21:47:05.194Z 5 Acked 1004 Write 2 Done Done Sent false
196742024-02-13T21:47:05.194Z 6 Acked 1005 Write 2 Done Done Sent false
196752024-02-13T21:47:05.194Z 7 Acked 1006 Read 8 Done Done Sent false
196762024-02-13T21:47:05.194Z 8 Acked 1007 Read 10 Done Done Sent false
196772024-02-13T21:47:05.194Z 9 Acked 1008 Read 2 Done Done Sent false
196782024-02-13T21:47:05.194Z 10 Acked 1009 Write 5 Done Done Sent false
196792024-02-13T21:47:05.194Z 11 Acked 1010 Write 9 Done Done Sent false
196802024-02-13T21:47:05.194Z 12 Acked 1011 Write 3 Done Done Sent false
196812024-02-13T21:47:05.194Z 13 Acked 1012 Read 8 Done Done Sent false
196822024-02-13T21:47:05.194Z 14 Acked 1013 Read 9 Done Done Sent false
196832024-02-13T21:47:05.195Z 15 Acked 1014 Write 10 Done Done Sent false
196842024-02-13T21:47:05.195Z 16 Acked 1015 Write 8 Done Done Sent false
196852024-02-13T21:47:05.195Z 17 Acked 1016 Read 7 Done Done Sent false
196862024-02-13T21:47:05.195Z 18 Acked 1017 Write 8 Done Done Sent false
196872024-02-13T21:47:05.195Z 19 Acked 1018 Write 4 Done Done Sent false
196882024-02-13T21:47:05.195Z 20 Acked 1019 Read 9 Done Done Sent false
196892024-02-13T21:47:05.195Z 21 Acked 1020 Write 10 Done Done Sent false
196902024-02-13T21:47:05.195Z 22 Acked 1021 Flush 0 Done Done Sent false
196912024-02-13T21:47:05.195Z 23 Acked 1022 Write 9 Done Done Sent false
196922024-02-13T21:47:05.195Z 24 Acked 1023 Read 10 Done Done Sent false
196932024-02-13T21:47:05.195Z 25 Acked 1024 Read 1 Done Done Sent false
196942024-02-13T21:47:05.195Z 26 Acked 1025 Read 6 Done Done Sent false
196952024-02-13T21:47:05.195Z 27 Acked 1026 Write 4 Done Done Sent false
196962024-02-13T21:47:05.196Z 28 Acked 1027 Read 2 Done Done Sent false
196972024-02-13T21:47:05.196Z 29 Acked 1028 Write 6 Sent Sent Sent false
196982024-02-13T21:47:05.196Z 30 Acked 1029 Write 2 Sent Sent Sent false
196992024-02-13T21:47:05.196Z STATES DS:0 DS:1 DS:2 TOTAL
197002024-02-13T21:47:05.196Z New 0 0 0 0
197012024-02-13T21:47:05.196Z Sent 2 2 30 34
197022024-02-13T21:47:05.196Z Done 28 28 0 56
197032024-02-13T21:47:05.196Z Skipped 0 0 0 0
197042024-02-13T21:47:05.196Z Error 0 0 0 0
197052024-02-13T21:47:05.196ZLast Flush: 1021 1021 0
197062024-02-13T21:47:05.196ZDownstairs last five completed:
197072024-02-13T21:47:05.196ZUpstairs last five completed: 30 29 28 27 26
197082024-02-13T21:47:05.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:03.866790017Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1866}
197092024-02-13T21:47:05.196ZWrote out file "/tmp/test_repair_verify.data"
197102024-02-13T21:47:05.197Z
197112024-02-13T21:47:05.200ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197122024-02-13T21:47:05.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197132024-02-13T21:47:05.211Z 0 000-019 179 179 179 328 328 328 F F F
197142024-02-13T21:47:05.211Z 1 020-039 179 179 179 328 328 328 F F F
197152024-02-13T21:47:05.211Z 2 040-059 182 182 180 332 332 329 F F F <---
197162024-02-13T21:47:05.211Z 3 060-079 178 178 178 326 326 326 F F F
197172024-02-13T21:47:05.211Z 4 080-099 182 182 177 332 332 324 F F F <---
197182024-02-13T21:47:05.211Z 5 100-119 172 172 172 314 314 314 F F F
197192024-02-13T21:47:05.211Z 6 120-139 178 178 178 326 326 326 F F F
197202024-02-13T21:47:05.211Z 7 140-159 178 178 178 325 325 325 F F F
197212024-02-13T21:47:05.211Z 8 160-179 182 182 179 332 332 328 F F F <---
197222024-02-13T21:47:05.211Z 9 180-199 177 177 177 324 324 324 T T F <---
197232024-02-13T21:47:05.211Z 10 200-219 176 176 176 321 321 321 F F F
197242024-02-13T21:47:05.211Z 11 220-239 173 173 173 315 315 315 F F F
197252024-02-13T21:47:05.216Z 12 240-259 182 182 180 332 332 329 F F F <---
197262024-02-13T21:47:05.216Z 13 260-279 182 182 178 332 332 326 F F F <---
197272024-02-13T21:47:05.216Z 14 280-299 181 181 181 331 331 331 T T F <---
197282024-02-13T21:47:05.216Z 15 300-319 158 158 158 297 297 297 F F F
197292024-02-13T21:47:05.216Z 16 320-339 176 176 176 322 322 322 F F F
197302024-02-13T21:47:05.216Z 17 340-359 181 181 181 331 331 331 F F F
197312024-02-13T21:47:05.216Z 18 360-379 182 182 181 332 332 331 F F F <---
197322024-02-13T21:47:05.216Z 19 380-399 180 180 180 330 330 330 F F F
197332024-02-13T21:47:05.216Z 20 400-419 182 182 181 332 332 331 T T F <---
197342024-02-13T21:47:05.216Z 21 420-439 174 174 174 317 317 317 F F F
197352024-02-13T21:47:05.216Z 22 440-459 182 182 181 332 332 331 F F F <---
197362024-02-13T21:47:05.216Z 23 460-479 182 182 173 332 332 315 F F F <---
197372024-02-13T21:47:05.216Z 24 480-499 181 181 181 331 331 331 F F F
197382024-02-13T21:47:05.216Z 25 500-519 175 175 175 318 318 318 F F F
197392024-02-13T21:47:05.217Z 26 520-539 176 176 176 321 321 321 F F F
197402024-02-13T21:47:05.217Z 27 540-559 181 181 181 331 331 331 T T F <---
197412024-02-13T21:47:05.217Z 28 560-579 182 182 178 332 332 327 F F F <---
197422024-02-13T21:47:05.217Z 29 580-599 182 182 178 332 332 326 F F F <---
197432024-02-13T21:47:05.217ZMax gen: 182, Max flush: 332
197442024-02-13T21:47:05.217ZError: Difference in extent metadata found!
197452024-02-13T21:47:05.217ZA Difference in extent metadata is expected here
197462024-02-13T21:47:05.217ZOn loop 91
197472024-02-13T21:47:05.217Z
197482024-02-13T21:47:05.220ZVerifying data now
197492024-02-13T21:47:09.900ZNo change after verify
197502024-02-13T21:47:09.903ZLoop: 91 Downstairs dump after verify (and repair):
197512024-02-13T21:47:09.915ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197522024-02-13T21:47:09.915Z 0 000-019 179 179 179 328 328 328 F F F
197532024-02-13T21:47:09.915Z 1 020-039 179 179 179 328 328 328 F F F
197542024-02-13T21:47:09.915Z 2 040-059 182 182 182 332 332 332 F F F
197552024-02-13T21:47:09.915Z 3 060-079 178 178 178 326 326 326 F F F
197562024-02-13T21:47:09.915Z 4 080-099 182 182 182 332 332 332 F F F
197572024-02-13T21:47:09.919Z 5 100-119 172 172 172 314 314 314 F F F
197582024-02-13T21:47:09.919Z 6 120-139 178 178 178 326 326 326 F F F
197592024-02-13T21:47:09.919Z 7 140-159 178 178 178 325 325 325 F F F
197602024-02-13T21:47:09.919Z 8 160-179 182 182 182 332 332 332 F F F
197612024-02-13T21:47:09.919Z 9 180-199 183 183 183 333 333 333 F F F
197622024-02-13T21:47:09.919Z 10 200-219 176 176 176 321 321 321 F F F
197632024-02-13T21:47:09.919Z 11 220-239 173 173 173 315 315 315 F F F
197642024-02-13T21:47:09.919Z 12 240-259 182 182 182 332 332 332 F F F
197652024-02-13T21:47:09.919Z 13 260-279 182 182 182 332 332 332 F F F
197662024-02-13T21:47:09.919Z 14 280-299 183 183 183 333 333 333 F F F
197672024-02-13T21:47:09.919Z 15 300-319 158 158 158 297 297 297 F F F
197682024-02-13T21:47:09.919Z 16 320-339 176 176 176 322 322 322 F F F
197692024-02-13T21:47:09.919Z 17 340-359 181 181 181 331 331 331 F F F
197702024-02-13T21:47:09.919Z 18 360-379 182 182 182 332 332 332 F F F
197712024-02-13T21:47:09.919Z 19 380-399 180 180 180 330 330 330 F F F
197722024-02-13T21:47:09.920Z 20 400-419 183 183 183 333 333 333 F F F
197732024-02-13T21:47:09.920Z 21 420-439 174 174 174 317 317 317 F F F
197742024-02-13T21:47:09.920Z 22 440-459 182 182 182 332 332 332 F F F
197752024-02-13T21:47:09.920Z 23 460-479 182 182 182 332 332 332 F F F
197762024-02-13T21:47:09.920Z 24 480-499 181 181 181 331 331 331 F F F
197772024-02-13T21:47:09.920Z 25 500-519 175 175 175 318 318 318 F F F
197782024-02-13T21:47:09.920Z 26 520-539 176 176 176 321 321 321 F F F
197792024-02-13T21:47:09.920Z 27 540-559 183 183 183 333 333 333 F F F
197802024-02-13T21:47:09.920Z 28 560-579 182 182 182 332 332 332 F F F
197812024-02-13T21:47:09.920Z 29 580-599 182 182 182 332 332 332 F F F
197822024-02-13T21:47:09.920ZMax gen: 183, Max flush: 333
197832024-02-13T21:47:09.920Z
197842024-02-13T21:47:09.920ZBegin loop 92
197852024-02-13T21:47:09.920ZDownstairs to restart: 0
197862024-02-13T21:47:09.936Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613356847Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
197872024-02-13T21:47:09.936Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",Crucible runtime is spawned
197882024-02-13T21:47:09.940Z\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613722054Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
197892024-02-13T21:47:09.940Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613767358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
197902024-02-13T21:47:09.940Z{"msg":"Crucible 620ad0c6-69a9-4f72-b988-b9fc27431d21 has session id: 0136dd6d-d56b-4a62-ad03-ee394eb55ff5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613807063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197912024-02-13T21:47:09.940Z{"msg":"Upstairs opts: Upstairs UUID: 620ad0c6-69a9-4f72-b988-b9fc27431d21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613857315Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197922024-02-13T21:47:09.940Z{"msg":"Crucible stats registered with UUID: 620ad0c6-69a9-4f72-b988-b9fc27431d21","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613896991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197932024-02-13T21:47:09.940Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613933417Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
197942024-02-13T21:47:09.940Z{"msg":"620ad0c6-69a9-4f72-b988-b9fc27431d21 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.613970214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197952024-02-13T21:47:09.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614013679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197962024-02-13T21:47:09.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614057352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197972024-02-13T21:47:09.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614089011Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197982024-02-13T21:47:09.940Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61417355Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
197992024-02-13T21:47:09.940Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614213716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198002024-02-13T21:47:09.941Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614241316Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198012024-02-13T21:47:09.941Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614268596Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198022024-02-13T21:47:09.941Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614318488Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198032024-02-13T21:47:09.941Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614359133Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198042024-02-13T21:47:09.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614384554Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198052024-02-13T21:47:09.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614409185Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198062024-02-13T21:47:09.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614440713Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"io task","client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198072024-02-13T21:47:09.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614953158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198082024-02-13T21:47:09.941Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.614989425Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198092024-02-13T21:47:09.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.615013526Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198102024-02-13T21:47:09.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.615035718Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198112024-02-13T21:47:09.941Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61555207Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198122024-02-13T21:47:09.941Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.615648765Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198132024-02-13T21:47:09.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61591145Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198142024-02-13T21:47:09.941Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.615950686Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198152024-02-13T21:47:09.941Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616088056Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198162024-02-13T21:47:09.941Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616139267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198172024-02-13T21:47:09.941Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616183501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198182024-02-13T21:47:09.941Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616218199Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198192024-02-13T21:47:09.941Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616241091Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198202024-02-13T21:47:09.941Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616255925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198212024-02-13T21:47:09.941Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616496188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198222024-02-13T21:47:09.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616936758Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198232024-02-13T21:47:09.945Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616966257Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198242024-02-13T21:47:09.946Z{"msg":"[0]R flush_numbers[0..12]: [328, 328, 332, 326, 332, 314, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.616982172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198252024-02-13T21:47:09.946Z{"msg":"[0]R generation[0..12]: [179, 179, 182, 178, 182, 172, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617017809Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198262024-02-13T21:47:09.946Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61704213Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198272024-02-13T21:47:09.946Z{"msg":"[1]R flush_numbers[0..12]: [328, 328, 332, 326, 332, 314, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61706976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198282024-02-13T21:47:09.946Z{"msg":"[1]R generation[0..12]: [179, 179, 182, 178, 182, 172, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617084974Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198292024-02-13T21:47:09.946Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617106187Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198302024-02-13T21:47:09.946Z{"msg":"[2]R flush_numbers[0..12]: [328, 328, 332, 326, 332, 314, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617121461Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198312024-02-13T21:47:09.946Z{"msg":"[2]R generation[0..12]: [179, 179, 182, 178, 182, 172, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
198322024-02-13T21:47:09.946Z,"time":"2024-02-13T21:47:08.617135966Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198332024-02-13T21:47:09.946Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61715271Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198342024-02-13T21:47:09.946Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617173312Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198352024-02-13T21:47:09.946Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617187327Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198362024-02-13T21:47:09.946Z{"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61720675Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198372024-02-13T21:47:09.946Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617220465Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"":"downstairs","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198382024-02-13T21:47:09.946Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617240248Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198392024-02-13T21:47:09.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617252753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"0","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198402024-02-13T21:47:09.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617272876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"1","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198412024-02-13T21:47:09.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617286281Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"client":"2","":"downstairs","session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198422024-02-13T21:47:09.946Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617299796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198432024-02-13T21:47:09.946Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617318729Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198442024-02-13T21:47:09.946Z{"msg":"620ad0c6-69a9-4f72-b988-b9fc27431d21 is now active with session: 0136dd6d-d56b-4a62-ad03-ee394eb55ff5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617332005Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198452024-02-13T21:47:09.946Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.61734422Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874,"session_id":"0136dd6d-d56b-4a62-ad03-ee394eb55ff5"}
198462024-02-13T21:47:09.946Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:08.617363253Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
198472024-02-13T21:47:09.946ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198482024-02-13T21:47:09.947ZLoading write count information from file "/tmp/test_repair_verify.data"
198492024-02-13T21:47:09.947ZRun Repair workload
198502024-02-13T21:47:09.947Z01/30 Write block 321 len 36864 data: 18 19 20 18 18 19 17 16 16
198512024-02-13T21:47:09.947Z02/30 Write block 118 len 4096 data: 10
198522024-02-13T21:47:09.947Z03/30 Read block 494 len 12288
198532024-02-13T21:47:09.947Z04/30 Write block 23 len 36864 data: 16 16 16 14 10 10 10 13 15
198542024-02-13T21:47:09.947Z05/30 Flush
198552024-02-13T21:47:09.950Z06/30 Write block 582 len 12288 data: 9 11 12
198562024-02-13T21:47:09.950Z07/30 Read block 285 len 40960
198572024-02-13T21:47:09.950Z08/30 Write block 306 len 4096 data: 7
198582024-02-13T21:47:09.950Z09/30 Write block 24 len 40960 data: 17 17 15 11 11 11 14 16 16 16
198592024-02-13T21:47:09.953Z10/30 Read block 301 len 40960
198602024-02-13T21:47:09.953Z11/30 Write block 551 len 32768 data: 21 18 17 16 15 16 15 12
198612024-02-13T21:47:09.956Z12/30 Write block 555 len 40960 data: 16 17 16 13 9 11 12 12 12 14
198622024-02-13T21:47:09.956Z13/30 Write block 50 len 24576 data: 13 12 12 12 14 16
198632024-02-13T21:47:09.957Z14/30 Flush
198642024-02-13T21:47:09.959Z15/30 Write block 456 len 32768 data: 19 18 18 19 21 20 17 15
198652024-02-13T21:47:09.959Z16/30 Read block 508 len 4096
198662024-02-13T21:47:09.968Z17/30 Write block 48 len 24576 data: 15 13 14 13 13 13
198672024-02-13T21:47:09.968Z18/30 Write block 451 len 12288 data: 17 15 17
198682024-02-13T21:47:09.968Z19/30 Write block 79 len 8192 data: 7 8
198692024-02-13T21:47:09.968Z20/30 Write block 7 len 28672 data: 12 11 13 13 16 16 16
198702024-02-13T21:47:09.968Z21/30 Write block 414 len 36864 data: 13 14 19 21 19 18 15 11 11
198712024-02-13T21:47:09.968Z22/30 Read block 475 len 20480
198722024-02-13T21:47:09.972Z23/30 Read block 496 len 12288
198732024-02-13T21:47:09.972Z24/30 Write block 28 len 12288 data: 12 12 15
198742024-02-13T21:47:09.972Z25/30 Write block 80 len 4096 data: 9
198752024-02-13T21:47:09.972Z26/30 Read block 208 len 28672
198762024-02-13T21:47:09.972Z27/30 Read block 313 len 12288
198772024-02-13T21:47:09.972Z28/30 Read block 508 len 20480
198782024-02-13T21:47:09.972Z29/30 Read block 246 len 24576
198792024-02-13T21:47:09.972Z30/30 Read block 228 len 8192
198802024-02-13T21:47:09.972Z----------------------------------------------------------------
198812024-02-13T21:47:09.972Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198822024-02-13T21:47:09.972ZStates: Active Active Active
198832024-02-13T21:47:09.972ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198842024-02-13T21:47:09.972Z 1 Acked 1000 Write 9 Done Done Done false
198852024-02-13T21:47:09.973Z 2 Acked 1001 Write 1 Done Done Done false
198862024-02-13T21:47:09.973Z 3 Acked 1002 Read 3 Sent Done Done false
198872024-02-13T21:47:09.973Z 4 Acked 1003 Write 9 Sent Done Done false
198882024-02-13T21:47:09.973Z 5 Acked 1004 Flush 0 Sent Done Done false
198892024-02-13T21:47:09.973Z 6 Acked 1005 Write 3 Sent Done Done false
198902024-02-13T21:47:09.973Z 7 Acked 1006 Read 10 Sent Done Done false
198912024-02-13T21:47:09.973Z 8 Acked 1007 Write 1 Sent Done Done false
198922024-02-13T21:47:09.973Z 9 Acked 1008 Write 10 Sent Done Done false
198932024-02-13T21:47:09.973Z 10 Acked 1009 Read 10 Sent Done Done false
198942024-02-13T21:47:09.973Z 11 Acked 1010 Write 8 Sent Done Done false
198952024-02-13T21:47:09.973Z 12 Acked 1011 Write 10 Sent Done Done false
198962024-02-13T21:47:09.973Z 13 Acked 1012 Write 6 Sent Done Done false
198972024-02-13T21:47:09.973Z 14 Acked 1013 Flush 0 Sent Done Done false
198982024-02-13T21:47:09.973Z 15 Acked 1014 Write 8 Sent Done Done false
198992024-02-13T21:47:09.973Z 16 Acked 1015 Read 1 Sent Done Done false
199002024-02-13T21:47:09.973Z 17 Acked 1016 Write 6 Sent Done Done false
199012024-02-13T21:47:09.973Z 18 Acked 1017 Write 3 Sent Done Done false
199022024-02-13T21:47:09.973Z 19 Acked 1018 Write 2 Sent Done Done false
199032024-02-13T21:47:09.973Z 20 Acked 1019 Write 7 Sent Done Done false
199042024-02-13T21:47:09.973Z 21 Acked 1020 Write 9 Sent Done Done false
199052024-02-13T21:47:09.973Z 22 Acked 1021 Read 5 Sent Done Done false
199062024-02-13T21:47:09.973Z 23 Acked 1022 Read 3 Sent Done Done false
199072024-02-13T21:47:09.973Z 24 Acked 1023 Write 3 Sent Done Done false
199082024-02-13T21:47:09.973Z 25 Acked 1024 Write 1 Sent Done Done false
199092024-02-13T21:47:09.973Z 26 Acked 1025 Read 7 Sent Done Done false
199102024-02-13T21:47:09.974Z 27 Acked 1026 Read 3 Sent Done Done false
199112024-02-13T21:47:09.974Z 28 Acked 1027 Read 5 Sent Done Done false
199122024-02-13T21:47:09.974Z 29 Acked 1028 Read 6 Sent Done Done false
199132024-02-13T21:47:09.974Z 30 Acked 1029 Read 2 Sent Sent Done false
199142024-02-13T21:47:09.974Z STATES DS:0 DS:1 DS:2 TOTAL
199152024-02-13T21:47:09.974Z New 0 0 0 0
199162024-02-13T21:47:09.974Z Sent 28 1 0 29
199172024-02-13T21:47:09.974Z Done 2 29 30 61
199182024-02-13T21:47:09.974Z Skipped 0 0 0 0
199192024-02-13T21:47:09.974Z Error 0 0 0 0
199202024-02-13T21:47:09.974ZLast Flush: 0 1013 1013
199212024-02-13T21:47:09.974ZDownstairs last five completed:
199222024-02-13T21:47:09.974ZUpstairs last five completed: 30 29 28 27 26
199232024-02-13T21:47:09.974Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:08.6484429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1874}
199242024-02-13T21:47:09.974ZWrote out file "/tmp/test_repair_verify.data"
199252024-02-13T21:47:09.977Z
199262024-02-13T21:47:09.980ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199272024-02-13T21:47:09.996ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199282024-02-13T21:47:09.996Z 0 000-019 179 179 179 328 328 328 F T T <---
199292024-02-13T21:47:09.996Z 1 020-039 179 184 184 328 335 335 F T T <---
199302024-02-13T21:47:09.996Z 2 040-059 182 184 184 332 335 335 F T T <---
199312024-02-13T21:47:09.996Z 3 060-079 178 178 178 326 326 326 F T T <---
199322024-02-13T21:47:09.996Z 4 080-099 182 182 182 332 332 332 F T T <---
199332024-02-13T21:47:09.996Z 5 100-119 172 184 184 314 334 334 T F F <---
199342024-02-13T21:47:09.996Z 6 120-139 178 178 178 326 326 326 F F F
199352024-02-13T21:47:09.996Z 7 140-159 178 178 178 325 325 325 F F F
199362024-02-13T21:47:09.996Z 8 160-179 182 182 182 332 332 332 F F F
199372024-02-13T21:47:09.996Z 9 180-199 183 183 183 333 333 333 F F F
199382024-02-13T21:47:09.996Z 10 200-219 176 176 176 321 321 321 F F F
199392024-02-13T21:47:10.000Z 11 220-239 173 173 173 315 315 315 F F F
199402024-02-13T21:47:10.000Z 12 240-259 182 182 182 332 332 332 F F F
199412024-02-13T21:47:10.000Z 13 260-279 182 182 182 332 332 332 F F F
199422024-02-13T21:47:10.000Z 14 280-299 183 183 183 333 333 333 F F F
199432024-02-13T21:47:10.000Z 15 300-319 158 184 184 297 335 335 F F F <---
199442024-02-13T21:47:10.000Z 16 320-339 176 184 184 322 334 334 T F F <---
199452024-02-13T21:47:10.000Z 17 340-359 181 181 181 331 331 331 F F F
199462024-02-13T21:47:10.000Z 18 360-379 182 182 182 332 332 332 F F F
199472024-02-13T21:47:10.000Z 19 380-399 180 180 180 330 330 330 F F F
199482024-02-13T21:47:10.000Z 20 400-419 183 183 183 333 333 333 F T T <---
199492024-02-13T21:47:10.000Z 21 420-439 174 174 174 317 317 317 F T T <---
199502024-02-13T21:47:10.000Z 22 440-459 182 182 182 332 332 332 F T T <---
199512024-02-13T21:47:10.000Z 23 460-479 182 182 182 332 332 332 F T T <---
199522024-02-13T21:47:10.000Z 24 480-499 181 181 181 331 331 331 F F F
199532024-02-13T21:47:10.000Z 25 500-519 175 175 175 318 318 318 F F F
199542024-02-13T21:47:10.000Z 26 520-539 176 176 176 321 321 321 F F F
199552024-02-13T21:47:10.000Z 27 540-559 183 184 184 333 335 335 F F F <---
199562024-02-13T21:47:10.000Z 28 560-579 182 184 184 332 335 335 F F F <---
199572024-02-13T21:47:10.000Z 29 580-599 182 184 184 332 335 335 F F F <---
199582024-02-13T21:47:10.001ZMax gen: 184, Max flush: 335
199592024-02-13T21:47:10.001ZError: Difference in extent metadata found!
199602024-02-13T21:47:10.001ZA Difference in extent metadata is expected here
199612024-02-13T21:47:10.001ZOn loop 92
199622024-02-13T21:47:10.001Z
199632024-02-13T21:47:10.005ZVerifying data now
199642024-02-13T21:47:15.728ZNo change after verify
199652024-02-13T21:47:15.728ZLoop: 92 Downstairs dump after verify (and repair):
199662024-02-13T21:47:15.743ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199672024-02-13T21:47:15.743Z 0 000-019 185 185 185 336 336 336 F F F
199682024-02-13T21:47:15.743Z 1 020-039 185 185 185 336 336 336 F F F
199692024-02-13T21:47:15.743Z 2 040-059 185 185 185 336 336 336 F F F
199702024-02-13T21:47:15.743Z 3 060-079 185 185 185 336 336 336 F F F
199712024-02-13T21:47:15.743Z 4 080-099 185 185 185 336 336 336 F F F
199722024-02-13T21:47:15.743Z 5 100-119 184 184 184 334 334 334 F F F
199732024-02-13T21:47:15.743Z 6 120-139 178 178 178 326 326 326 F F F
199742024-02-13T21:47:15.743Z 7 140-159 178 178 178 325 325 325 F F F
199752024-02-13T21:47:15.746Z 8 160-179 182 182 182 332 332 332 F F F
199762024-02-13T21:47:15.746Z 9 180-199 183 183 183 333 333 333 F F F
199772024-02-13T21:47:15.747Z 10 200-219 176 176 176 321 321 321 F F F
199782024-02-13T21:47:15.747Z 11 220-239 173 173 173 315 315 315 F F F
199792024-02-13T21:47:15.747Z 12 240-259 182 182 182 332 332 332 F F F
199802024-02-13T21:47:15.747Z 13 260-279 182 182 182 332 332 332 F F F
199812024-02-13T21:47:15.747Z 14 280-299 183 183 183 333 333 333 F F F
199822024-02-13T21:47:15.747Z 15 300-319 184 184 184 335 335 335 F F F
199832024-02-13T21:47:15.747Z 16 320-339 184 184 184 334 334 334 F F F
199842024-02-13T21:47:15.747Z 17 340-359 181 181 181 331 331 331 F F F
199852024-02-13T21:47:15.747Z 18 360-379 182 182 182 332 332 332 F F F
199862024-02-13T21:47:15.747Z 19 380-399 180 180 180 330 330 330 F F F
199872024-02-13T21:47:15.747Z 20 400-419 185 185 185 336 336 336 F F F
199882024-02-13T21:47:15.747Z 21 420-439 185 185 185 336 336 336 F F F
199892024-02-13T21:47:15.747Z 22 440-459 185 185 185 336 336 336 F F F
199902024-02-13T21:47:15.747Z 23 460-479 185 185 185 336 336 336 F F F
199912024-02-13T21:47:15.747Z 24 480-499 181 181 181 331 331 331 F F F
199922024-02-13T21:47:15.747Z 25 500-519 175 175 175 318 318 318 F F F
199932024-02-13T21:47:15.747Z 26 520-539 176 176 176 321 321 321 F F F
199942024-02-13T21:47:15.747Z 27 540-559 184 184 184 335 335 335 F F F
199952024-02-13T21:47:15.747Z 28 560-579 184 184 184 335 335 335 F F F
199962024-02-13T21:47:15.747Z 29 580-599 184 184 184 335 335 335 F F F
199972024-02-13T21:47:15.747ZMax gen: 185, Max flush: 336
199982024-02-13T21:47:15.747Z
199992024-02-13T21:47:15.747ZBegin loop 93
200002024-02-13T21:47:15.747ZDownstairs to restart: 2
200012024-02-13T21:47:15.766Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.440643611Z","Crucible runtime is spawned
200022024-02-13T21:47:15.766Zhostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
200032024-02-13T21:47:15.770Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441012997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
200042024-02-13T21:47:15.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441060029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
200052024-02-13T21:47:15.770Z{"msg":"Crucible 7141ab83-13ff-4d49-9c9a-a8e550e91833 has session id: cdd95892-5b42-41f5-8ad5-18e4a6c00963","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441084321Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200062024-02-13T21:47:15.770Z{"msg":"Upstairs opts: Upstairs UUID: 7141ab83-13ff-4d49-9c9a-a8e550e91833, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441128944Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200072024-02-13T21:47:15.770Z{"msg":"Crucible stats registered with UUID: 7141ab83-13ff-4d49-9c9a-a8e550e91833","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441165471Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200082024-02-13T21:47:15.770Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441190052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
200092024-02-13T21:47:15.770Z{"msg":"7141ab83-13ff-4d49-9c9a-a8e550e91833 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441221251Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200102024-02-13T21:47:15.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44125249Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200112024-02-13T21:47:15.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441276661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200122024-02-13T21:47:15.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441319745Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200132024-02-13T21:47:15.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441351184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200142024-02-13T21:47:15.771Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44138949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200152024-02-13T21:47:15.771Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44141704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200162024-02-13T21:47:15.771Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441454097Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200172024-02-13T21:47:15.771Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441481407Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200182024-02-13T21:47:15.771Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441515644Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200192024-02-13T21:47:15.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441541585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200202024-02-13T21:47:15.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44158165Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200212024-02-13T21:47:15.771Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.441607881Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"io task","client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200222024-02-13T21:47:15.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.442307457Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200232024-02-13T21:47:15.771Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44235356Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200242024-02-13T21:47:15.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44238255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200252024-02-13T21:47:15.771Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.442412309Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200262024-02-13T21:47:15.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44293269Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200272024-02-13T21:47:15.771Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.442979423Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200282024-02-13T21:47:15.771Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443019019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200292024-02-13T21:47:15.771Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443058575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200302024-02-13T21:47:15.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443615533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200312024-02-13T21:47:15.771Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.44365148Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200322024-02-13T21:47:15.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443668633Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200332024-02-13T21:47:15.771Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443701681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200342024-02-13T21:47:15.772Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443801655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200352024-02-13T21:47:15.772Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443849888Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200362024-02-13T21:47:15.772Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.443884415Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200372024-02-13T21:47:15.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444251872Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200382024-02-13T21:47:15.776Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444288588Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200392024-02-13T21:47:15.776Z{"msg":"[0]R flush_numbers[0..12]: [336, 336, 336, 336, 336, 334, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444304812Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200402024-02-13T21:47:15.776Z{"msg":"[0]R generation[0..12]: [185, 185, 185, 185, 185, 184, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444320077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200412024-02-13T21:47:15.776Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444341799Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200422024-02-13T21:47:15.776Z{"msg":"[1]R flush_numbers[0..12]: [336, 336, 336, 336, 336, 334, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444356614Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200432024-02-13T21:47:15.776Z{"msg":"[1]R generation[0..12]: [185, 185, 185, 185, 185, 184, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444377546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200442024-02-13T21:47:15.776Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444392261Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200452024-02-13T21:47:15.776Z{"msg":"[2]R flush_numbers[0..12]: [336, 336, 336, 336, 336, 334, 326, 325, 332, 333, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444418301Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200462024-02-13T21:47:15.776Z{"msg":"[2]R generation[0..12]: [185, 185, 185, 185, 185, 184, 178, 178, 182, 183, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444439474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
200472024-02-13T21:47:15.776Zsession_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200482024-02-13T21:47:15.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444467983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200492024-02-13T21:47:15.776Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444481808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200502024-02-13T21:47:15.776Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.4445034Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200512024-02-13T21:47:15.776Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444516906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200522024-02-13T21:47:15.776Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444541826Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"":"downstairs","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200532024-02-13T21:47:15.776Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444562859Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200542024-02-13T21:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444593987Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"0","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200552024-02-13T21:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444609942Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"1","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200562024-02-13T21:47:15.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444634203Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"client":"2","":"downstairs","session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200572024-02-13T21:47:15.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444667331Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200582024-02-13T21:47:15.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444701998Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200592024-02-13T21:47:15.777Z{"msg":"7141ab83-13ff-4d49-9c9a-a8e550e91833 is now active with session: cdd95892-5b42-41f5-8ad5-18e4a6c00963","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444736796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200602024-02-13T21:47:15.777Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444768684Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882,"session_id":"cdd95892-5b42-41f5-8ad5-18e4a6c00963"}
200612024-02-13T21:47:15.777Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:14.444802512Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
200622024-02-13T21:47:15.777ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200632024-02-13T21:47:15.777ZLoading write count information from file "/tmp/test_repair_verify.data"
200642024-02-13T21:47:15.777ZRun Repair workload
200652024-02-13T21:47:15.777Z01/30 Flush
200662024-02-13T21:47:15.777Z02/30 Write block 512 len 32768 data: 14 14 18 17 18 21 21 23
200672024-02-13T21:47:15.777Z03/30 Read block 513 len 32768
200682024-02-13T21:47:15.777Z04/30 Write block 519 len 24576 data: 24 23 21 19 19 14
200692024-02-13T21:47:15.777Z05/30 Read block 581 len 36864
200702024-02-13T21:47:15.780Z06/30 Write block 194 len 12288 data: 16 14 15
200712024-02-13T21:47:15.780Z07/30 Read block 580 len 8192
200722024-02-13T21:47:15.780Z08/30 Write block 301 len 20480 data: 10 8 11 11 8
200732024-02-13T21:47:15.780Z09/30 Read block 145 len 16384
200742024-02-13T21:47:15.781Z10/30 Write block 75 len 12288 data: 15 14 12
200752024-02-13T21:47:15.781Z11/30 Read block 367 len 12288
200762024-02-13T21:47:15.783Z12/30 Write block 449 len 28672 data: 19 19 18 16 18 17 16
200772024-02-13T21:47:15.784Z13/30 Write block 298 len 20480 data: 14 11 10 11 9
200782024-02-13T21:47:15.784Z14/30 Read block 247 len 12288
200792024-02-13T21:47:15.784Z15/30 Write block 285 len 4096 data: 12
200802024-02-13T21:47:15.784Z16/30 Write block 488 len 12288 data: 13 13 11
200812024-02-13T21:47:15.787Z17/30 Read block 24 len 4096
200822024-02-13T21:47:15.787Z18/30 Write block 475 len 8192 data: 19 16
200832024-02-13T21:47:15.787Z19/30 Write block 165 len 24576 data: 10 13 10 8 8 6
200842024-02-13T21:47:15.787Z20/30 Write block 535 len 32768 data: 16 14 15 18 21 24 20 20
200852024-02-13T21:47:15.787Z21/30 Read block 411 len 24576
200862024-02-13T21:47:15.790Z22/30 Write block 497 len 24576 data: 15 14 12 13 9 11
200872024-02-13T21:47:15.790Z23/30 Flush
200882024-02-13T21:47:15.794Z24/30 Write block 384 len 36864 data: 16 16 16 15 15 14 12 11 11
200892024-02-13T21:47:15.794Z25/30 Flush
200902024-02-13T21:47:15.797Z26/30 Write block 585 len 4096 data: 13
200912024-02-13T21:47:15.800Z27/30 Write block 579 len 4096 data: 8
200922024-02-13T21:47:15.800Z28/30 Read block 471 len 36864
200932024-02-13T21:47:15.800Z29/30 Write block 160 len 28672 data: 12 15 15 13 11 11 14
200942024-02-13T21:47:15.800Z30/30 Read block 47 len 32768
200952024-02-13T21:47:15.806Z----------------------------------------------------------------
200962024-02-13T21:47:15.806Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:29
200972024-02-13T21:47:15.806ZStates: Active Active Active
200982024-02-13T21:47:15.806ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200992024-02-13T21:47:15.807Z 2 Acked 1001 Write 8 Done Done Done false
201002024-02-13T21:47:15.807Z 3 Acked 1002 Read 8 Done Done Done false
201012024-02-13T21:47:15.807Z 4 Acked 1003 Write 6 Done Done Done false
201022024-02-13T21:47:15.807Z 5 Acked 1004 Read 9 Done Done Done false
201032024-02-13T21:47:15.807Z 6 Acked 1005 Write 3 Done Done Done false
201042024-02-13T21:47:15.807Z 7 Acked 1006 Read 2 Done Done Sent false
201052024-02-13T21:47:15.807Z 8 Acked 1007 Write 5 Done Done Sent false
201062024-02-13T21:47:15.807Z 9 Acked 1008 Read 4 Done Done Sent false
201072024-02-13T21:47:15.807Z 10 Acked 1009 Write 3 Done Done Sent false
201082024-02-13T21:47:15.807Z 11 Acked 1010 Read 3 Done Done Sent false
201092024-02-13T21:47:15.807Z 12 Acked 1011 Write 7 Done Done Sent false
201102024-02-13T21:47:15.807Z 13 Acked 1012 Write 5 Done Done Sent false
201112024-02-13T21:47:15.807Z 14 Acked 1013 Read 3 Done Done Sent false
201122024-02-13T21:47:15.807Z 15 Acked 1014 Write 1 Done Done Sent false
201132024-02-13T21:47:15.807Z 16 Acked 1015 Write 3 Done Done Sent false
201142024-02-13T21:47:15.807Z 17 Acked 1016 Read 1 Done Done Sent false
201152024-02-13T21:47:15.807Z 18 Acked 1017 Write 2 Done Done Sent false
201162024-02-13T21:47:15.807Z 19 Acked 1018 Write 6 Done Done Sent false
201172024-02-13T21:47:15.807Z 20 Acked 1019 Write 8 Done Done Sent false
201182024-02-13T21:47:15.807Z 21 Acked 1020 Read 6 Done Done Sent false
201192024-02-13T21:47:15.807Z 22 Acked 1021 Write 6 Done Done Sent false
201202024-02-13T21:47:15.807Z 23 Acked 1022 Flush 0 Done Done Sent false
201212024-02-13T21:47:15.807Z 24 Acked 1023 Write 9 Done Done Sent false
201222024-02-13T21:47:15.808Z 25 Acked 1024 Flush 0 Done Done Sent false
201232024-02-13T21:47:15.808Z 26 Acked 1025 Write 1 Done Done Sent false
201242024-02-13T21:47:15.808Z 27 Acked 1026 Write 1 Done Done Sent false
201252024-02-13T21:47:15.808Z 28 Acked 1027 Read 9 Done Done Sent false
201262024-02-13T21:47:15.808Z 29 Acked 1028 Write 7 Done Done Sent false
201272024-02-13T21:47:15.808Z 30 Acked 1029 Read 8 Sent Done Sent false
201282024-02-13T21:47:15.808Z STATES DS:0 DS:1 DS:2 TOTAL
201292024-02-13T21:47:15.808Z New 0 0 0 0
201302024-02-13T21:47:15.808Z Sent 1 0 24 25
201312024-02-13T21:47:15.808Z Done 28 29 5 62
201322024-02-13T21:47:15.808Z Skipped 0 0 0 0
201332024-02-13T21:47:15.808Z Error 0 0 0 0
201342024-02-13T21:47:15.808ZLast Flush: 1024 1024 1000
201352024-02-13T21:47:15.808ZDownstairs last five completed: 1000
201362024-02-13T21:47:15.808ZUpstairs last five completed: 30 29 28 27 26
201372024-02-13T21:47:15.808Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:14.476523466Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1882}
201382024-02-13T21:47:15.808ZWrote out file "/tmp/test_repair_verify.data"
201392024-02-13T21:47:15.811Z
201402024-02-13T21:47:15.811ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201412024-02-13T21:47:15.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201422024-02-13T21:47:15.825Z 0 000-019 185 185 185 336 336 336 F F F
201432024-02-13T21:47:15.825Z 1 020-039 185 185 185 336 336 336 F F F
201442024-02-13T21:47:15.825Z 2 040-059 185 185 185 336 336 336 F F F
201452024-02-13T21:47:15.825Z 3 060-079 186 186 185 338 338 336 F F F <---
201462024-02-13T21:47:15.825Z 4 080-099 185 185 185 336 336 336 F F F
201472024-02-13T21:47:15.825Z 5 100-119 184 184 184 334 334 334 F F F
201482024-02-13T21:47:15.825Z 6 120-139 178 178 178 326 326 326 F F F
201492024-02-13T21:47:15.825Z 7 140-159 178 178 178 325 325 325 F F F
201502024-02-13T21:47:15.825Z 8 160-179 186 186 182 338 338 332 T T F <---
201512024-02-13T21:47:15.825Z 9 180-199 186 186 183 338 338 333 F F T <---
201522024-02-13T21:47:15.825Z 10 200-219 176 176 176 321 321 321 F F F
201532024-02-13T21:47:15.825Z 11 220-239 173 173 173 315 315 315 F F F
201542024-02-13T21:47:15.829Z 12 240-259 182 182 182 332 332 332 F F F
201552024-02-13T21:47:15.829Z 13 260-279 182 182 182 332 332 332 F F F
201562024-02-13T21:47:15.829Z 14 280-299 186 186 183 338 338 333 F F F <---
201572024-02-13T21:47:15.829Z 15 300-319 186 186 184 338 338 335 F F F <---
201582024-02-13T21:47:15.829Z 16 320-339 184 184 184 334 334 334 F F F
201592024-02-13T21:47:15.829Z 17 340-359 181 181 181 331 331 331 F F F
201602024-02-13T21:47:15.829Z 18 360-379 182 182 182 332 332 332 F F F
201612024-02-13T21:47:15.829Z 19 380-399 186 186 180 339 339 330 F F F <---
201622024-02-13T21:47:15.829Z 20 400-419 185 185 185 336 336 336 F F F
201632024-02-13T21:47:15.829Z 21 420-439 185 185 185 336 336 336 F F F
201642024-02-13T21:47:15.829Z 22 440-459 186 186 185 338 338 336 F F F <---
201652024-02-13T21:47:15.829Z 23 460-479 186 186 185 338 338 336 F F F <---
201662024-02-13T21:47:15.829Z 24 480-499 186 186 181 338 338 331 F F F <---
201672024-02-13T21:47:15.830Z 25 500-519 186 186 175 338 338 318 F F T <---
201682024-02-13T21:47:15.830Z 26 520-539 186 186 176 338 338 321 F F T <---
201692024-02-13T21:47:15.830Z 27 540-559 186 186 184 338 338 335 F F F <---
201702024-02-13T21:47:15.830Z 28 560-579 184 184 184 335 335 335 T T F <---
201712024-02-13T21:47:15.830Z 29 580-599 184 184 184 335 335 335 T T F <---
201722024-02-13T21:47:15.830ZMax gen: 186, Max flush: 339
201732024-02-13T21:47:15.830ZError: Difference in extent metadata found!
201742024-02-13T21:47:15.830ZA Difference in extent metadata is expected here
201752024-02-13T21:47:15.830ZOn loop 93
201762024-02-13T21:47:15.830Z
201772024-02-13T21:47:15.833ZVerifying data now
201782024-02-13T21:47:21.513ZNo change after verify
201792024-02-13T21:47:21.514ZLoop: 93 Downstairs dump after verify (and repair):
201802024-02-13T21:47:21.528ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201812024-02-13T21:47:21.529Z 0 000-019 185 185 185 336 336 336 F F F
201822024-02-13T21:47:21.529Z 1 020-039 185 185 185 336 336 336 F F F
201832024-02-13T21:47:21.529Z 2 040-059 185 185 185 336 336 336 F F F
201842024-02-13T21:47:21.529Z 3 060-079 186 186 186 338 338 338 F F F
201852024-02-13T21:47:21.529Z 4 080-099 185 185 185 336 336 336 F F F
201862024-02-13T21:47:21.529Z 5 100-119 184 184 184 334 334 334 F F F
201872024-02-13T21:47:21.529Z 6 120-139 178 178 178 326 326 326 F F F
201882024-02-13T21:47:21.529Z 7 140-159 178 178 178 325 325 325 F F F
201892024-02-13T21:47:21.529Z 8 160-179 187 187 187 340 340 340 F F F
201902024-02-13T21:47:21.529Z 9 180-199 186 186 186 338 338 338 F F F
201912024-02-13T21:47:21.529Z 10 200-219 176 176 176 321 321 321 F F F
201922024-02-13T21:47:21.529Z 11 220-239 173 173 173 315 315 315 F F F
201932024-02-13T21:47:21.532Z 12 240-259 182 182 182 332 332 332 F F F
201942024-02-13T21:47:21.532Z 13 260-279 182 182 182 332 332 332 F F F
201952024-02-13T21:47:21.532Z 14 280-299 186 186 186 338 338 338 F F F
201962024-02-13T21:47:21.532Z 15 300-319 186 186 186 338 338 338 F F F
201972024-02-13T21:47:21.532Z 16 320-339 184 184 184 334 334 334 F F F
201982024-02-13T21:47:21.532Z 17 340-359 181 181 181 331 331 331 F F F
201992024-02-13T21:47:21.533Z 18 360-379 182 182 182 332 332 332 F F F
202002024-02-13T21:47:21.533Z 19 380-399 186 186 186 339 339 339 F F F
202012024-02-13T21:47:21.533Z 20 400-419 185 185 185 336 336 336 F F F
202022024-02-13T21:47:21.533Z 21 420-439 185 185 185 336 336 336 F F F
202032024-02-13T21:47:21.533Z 22 440-459 186 186 186 338 338 338 F F F
202042024-02-13T21:47:21.533Z 23 460-479 186 186 186 338 338 338 F F F
202052024-02-13T21:47:21.533Z 24 480-499 186 186 186 338 338 338 F F F
202062024-02-13T21:47:21.533Z 25 500-519 186 186 186 338 338 338 F F F
202072024-02-13T21:47:21.533Z 26 520-539 186 186 186 338 338 338 F F F
202082024-02-13T21:47:21.533Z 27 540-559 186 186 186 338 338 338 F F F
202092024-02-13T21:47:21.533Z 28 560-579 187 187 187 340 340 340 F F F
202102024-02-13T21:47:21.533Z 29 580-599 187 187 187 340 340 340 F F F
202112024-02-13T21:47:21.533ZMax gen: 187, Max flush: 340
202122024-02-13T21:47:21.533Z
202132024-02-13T21:47:21.533ZBegin loop 94
202142024-02-13T21:47:21.533ZDownstairs to restart: 0
202152024-02-13T21:47:21.551Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.222674899Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
202162024-02-13T21:47:21.557Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.222941822Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
202172024-02-13T21:47:21.557ZCrucible runtime is spawned
202182024-02-13T21:47:21.557Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.222993863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
202192024-02-13T21:47:21.557Z{"msg":"Crucible f9e5518d-78d1-4848-af78-9045c71b8e69 has session id: aca6f567-7b5c-462c-b9da-e3eb3324fe77","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223035038Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202202024-02-13T21:47:21.557Z{"msg":"Upstairs opts: Upstairs UUID: f9e5518d-78d1-4848-af78-9045c71b8e69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223070595Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202212024-02-13T21:47:21.557Z{"msg":"Crucible stats registered with UUID: f9e5518d-78d1-4848-af78-9045c71b8e69","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223099374Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202222024-02-13T21:47:21.557Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223112949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
202232024-02-13T21:47:21.557Z{"msg":"f9e5518d-78d1-4848-af78-9045c71b8e69 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223205866Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202242024-02-13T21:47:21.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223231666Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202252024-02-13T21:47:21.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223247751Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202262024-02-13T21:47:21.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.22327652Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202272024-02-13T21:47:21.557Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223297953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202282024-02-13T21:47:21.557Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223337408Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202292024-02-13T21:47:21.558Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223361339Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202302024-02-13T21:47:21.558Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223382472Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202312024-02-13T21:47:21.558Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.22341497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202322024-02-13T21:47:21.558Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.22344105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202332024-02-13T21:47:21.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223478807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202342024-02-13T21:47:21.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223542384Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202352024-02-13T21:47:21.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.223570993Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"io task","client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202362024-02-13T21:47:21.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224180622Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202372024-02-13T21:47:21.558Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224216199Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202382024-02-13T21:47:21.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224248697Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202392024-02-13T21:47:21.558Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224281326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202402024-02-13T21:47:21.558Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224907019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202412024-02-13T21:47:21.558Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.224952642Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202422024-02-13T21:47:21.558Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225263189Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202432024-02-13T21:47:21.558Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225340572Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202442024-02-13T21:47:21.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225450821Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202452024-02-13T21:47:21.558Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.22548156Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202462024-02-13T21:47:21.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225497045Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202472024-02-13T21:47:21.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225511499Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202482024-02-13T21:47:21.558Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225526014Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202492024-02-13T21:47:21.558Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225539709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202502024-02-13T21:47:21.558Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.225833233Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202512024-02-13T21:47:21.558Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226270284Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202522024-02-13T21:47:21.559Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226305482Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202532024-02-13T21:47:21.559Z{"msg":"[0]R flush_numbers[0..12]: [336, 336, 336, 338, 336, 334, 326, 325, 340, 338, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226323305Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202542024-02-13T21:47:21.559Z{"msg":"[0]R generation[0..12]: [185, 185, 185, 186, 185, 184, 178, 178, 187, 186, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226345397Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202552024-02-13T21:47:21.559Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226359932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202562024-02-13T21:47:21.559Z{"msg":"[1]R flush_numbers[0..12]: [336, 336, 336, 338, 336, 334, 326, 325, 340, 338, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226380654Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202572024-02-13T21:47:21.559Z{"msg":"[1]R generation[0..12]: [185, 185, 185, 186, 185, 184, 178, 178, 187, 186, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226394709Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202582024-02-13T21:47:21.559Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226415362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202592024-02-13T21:47:21.559Z{"msg":"[2]R flush_numbers[0..12]: [336, 336, 336, 338, 336, 334, 326, 325, 340, 338, 321, 315]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226429626Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202602024-02-13T21:47:21.559Z{"msg":"[2]R generation[0..12]: [185, 185, 185, 186, 185, 184, 178, 178, 187, 186, 176, 173]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226454997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202612024-02-13T21:47:21.559Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
202622024-02-13T21:47:21.559Z,"time":"2024-02-13T21:47:20.226488545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202632024-02-13T21:47:21.559Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226520983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202642024-02-13T21:47:21.559Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226553182Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202652024-02-13T21:47:21.559Z{"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226568586Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202662024-02-13T21:47:21.559Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226589019Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"":"downstairs","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202672024-02-13T21:47:21.559Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226603803Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202682024-02-13T21:47:21.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226635242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"0","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202692024-02-13T21:47:21.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226670949Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"1","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202702024-02-13T21:47:21.559Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.2266944Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"client":"2","":"downstairs","session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202712024-02-13T21:47:21.559Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226712454Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202722024-02-13T21:47:21.559Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226744862Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202732024-02-13T21:47:21.559Z{"msg":"f9e5518d-78d1-4848-af78-9045c71b8e69 is now active with session: aca6f567-7b5c-462c-b9da-e3eb3324fe77","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226780539Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202742024-02-13T21:47:21.559Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226803221Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890,"session_id":"aca6f567-7b5c-462c-b9da-e3eb3324fe77"}
202752024-02-13T21:47:21.559Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:20.226820485Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
202762024-02-13T21:47:21.559ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202772024-02-13T21:47:21.559ZLoading write count information from file "/tmp/test_repair_verify.data"
202782024-02-13T21:47:21.560ZRun Repair workload
202792024-02-13T21:47:21.560Z01/30 Write block 535 len 24576 data: 17 15 16 19 22 25
202802024-02-13T21:47:21.563Z02/30 Write block 183 len 36864 data: 13 14 17 14 13 15 14 13 12
202812024-02-13T21:47:21.563Z03/30 Write block 561 len 24576 data: 13 13 13 15 14 13
202822024-02-13T21:47:21.563Z04/30 Write block 205 len 8192 data: 10 5
202832024-02-13T21:47:21.563Z05/30 Read block 531 len 32768
202842024-02-13T21:47:21.563Z06/30 Write block 345 len 36864 data: 14 13 14 14 12 12 13 15 16
202852024-02-13T21:47:21.564Z07/30 Write block 351 len 4096 data: 14
202862024-02-13T21:47:21.564Z08/30 Write block 271 len 28672 data: 14 16 18 20 21 19 21
202872024-02-13T21:47:21.564Z09/30 Write block 38 len 36864 data: 14 13 15 14 12 12 11 11 13
202882024-02-13T21:47:21.564Z10/30 Write block 549 len 4096 data: 15
202892024-02-13T21:47:21.567Z11/30 Write block 10 len 40960 data: 14 17 17 17 17 17 17 16 17 14
202902024-02-13T21:47:21.567Z12/30 Write block 397 len 28672 data: 10 11 10 13 13 15 16
202912024-02-13T21:47:21.567Z13/30 Read block 304 len 8192
202922024-02-13T21:47:21.567Z14/30 Read block 276 len 40960
202932024-02-13T21:47:21.567Z15/30 Write block 115 len 12288 data: 11 13 14
202942024-02-13T21:47:21.570Z16/30 Write block 246 len 24576 data: 10 10 10 15 17 19
202952024-02-13T21:47:21.571Z17/30 Read block 378 len 40960
202962024-02-13T21:47:21.571Z18/30 Read block 247 len 28672
202972024-02-13T21:47:21.571Z19/30 Read block 125 len 40960
202982024-02-13T21:47:21.573Z20/30 Write block 540 len 4096 data: 26
202992024-02-13T21:47:21.574Z21/30 Read block 263 len 20480
203002024-02-13T21:47:21.574Z22/30 Read block 573 len 40960
203012024-02-13T21:47:21.576Z23/30 Write block 476 len 28672 data: 17 14 14 11 11 13 15
203022024-02-13T21:47:21.577Z24/30 Write block 426 len 16384 data: 13 14 16 20
203032024-02-13T21:47:21.577Z25/30 Read block 347 len 28672
203042024-02-13T21:47:21.577Z26/30 Write block 238 len 20480 data: 13 13 14 14 15
203052024-02-13T21:47:21.583Z27/30 Read block 111 len 36864
203062024-02-13T21:47:21.583Z28/30 Write block 67 len 4096 data: 15
203072024-02-13T21:47:21.583Z29/30 Read block 497 len 32768
203082024-02-13T21:47:21.586Z30/30 Write block 153 len 20480 data: 14 14 15 13 13
203092024-02-13T21:47:21.587Z----------------------------------------------------------------
203102024-02-13T21:47:21.587Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
203112024-02-13T21:47:21.587ZStates: Active Active Active
203122024-02-13T21:47:21.587ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203132024-02-13T21:47:21.587Z 1 Acked 1000 Write 6 Done Done Done false
203142024-02-13T21:47:21.587Z 2 Acked 1001 Write 9 Done Done Done false
203152024-02-13T21:47:21.587Z 3 Acked 1002 Write 6 Sent Done Done false
203162024-02-13T21:47:21.587Z 4 Acked 1003 Write 2 Sent Done Done false
203172024-02-13T21:47:21.587Z 5 Acked 1004 Read 8 Sent Done Done false
203182024-02-13T21:47:21.587Z 6 Acked 1005 Write 9 Sent Done Done false
203192024-02-13T21:47:21.587Z 7 Acked 1006 Write 1 Sent Done Done false
203202024-02-13T21:47:21.587Z 8 Acked 1007 Write 7 Sent Done Done false
203212024-02-13T21:47:21.587Z 9 Acked 1008 Write 9 Sent Done Done false
203222024-02-13T21:47:21.587Z 10 Acked 1009 Write 1 Sent Done Done false
203232024-02-13T21:47:21.587Z 11 Acked 1010 Write 10 Sent Done Done false
203242024-02-13T21:47:21.587Z 12 Acked 1011 Write 7 Sent Done Done false
203252024-02-13T21:47:21.587Z 13 Acked 1012 Read 2 Sent Done Done false
203262024-02-13T21:47:21.587Z 14 Acked 1013 Read 10 Sent Done Done false
203272024-02-13T21:47:21.587Z 15 Acked 1014 Write 3 Sent Done Done false
203282024-02-13T21:47:21.587Z 16 Acked 1015 Write 6 Sent Done Done false
203292024-02-13T21:47:21.587Z 17 Acked 1016 Read 10 Sent Done Done false
203302024-02-13T21:47:21.587Z 18 Acked 1017 Read 7 Sent Done Done false
203312024-02-13T21:47:21.587Z 19 Acked 1018 Read 10 Sent Done Done false
203322024-02-13T21:47:21.587Z 20 Acked 1019 Write 1 Sent Done Done false
203332024-02-13T21:47:21.588Z 21 Acked 1020 Read 5 Sent Done Done false
203342024-02-13T21:47:21.588Z 22 Acked 1021 Read 10 Sent Done Done false
203352024-02-13T21:47:21.588Z 23 Acked 1022 Write 7 Sent Done Done false
203362024-02-13T21:47:21.588Z 24 Acked 1023 Write 4 Sent Done Done false
203372024-02-13T21:47:21.588Z 25 Acked 1024 Read 7 Sent Done Done false
203382024-02-13T21:47:21.588Z 26 Acked 1025 Write 5 Sent Done Done false
203392024-02-13T21:47:21.588Z 27 Acked 1026 Read 9 Sent Done Done false
203402024-02-13T21:47:21.588Z 28 Acked 1027 Write 1 Sent Done Done false
203412024-02-13T21:47:21.588Z 29 Acked 1028 Read 8 Sent Done Done false
203422024-02-13T21:47:21.588Z 30 Acked 1029 Write 5 Sent Sent Sent false
203432024-02-13T21:47:21.588Z STATES DS:0 DS:1 DS:2 TOTAL
203442024-02-13T21:47:21.588Z New 0 0 0 0
203452024-02-13T21:47:21.588Z Sent 28 1 1 30
203462024-02-13T21:47:21.588Z Done 2 29 29 60
203472024-02-13T21:47:21.588Z Skipped 0 0 0 0
203482024-02-13T21:47:21.588Z Error 0 0 0 0
203492024-02-13T21:47:21.588ZLast Flush: 0 0 0
203502024-02-13T21:47:21.588ZDownstairs last five completed:
203512024-02-13T21:47:21.588ZUpstairs last five completed: 30 29 28 27 26
203522024-02-13T21:47:21.588Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:20.254017559Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1890}
203532024-02-13T21:47:21.588ZWrote out file "/tmp/test_repair_verify.data"
203542024-02-13T21:47:21.588Z
203552024-02-13T21:47:21.592ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203562024-02-13T21:47:21.611ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203572024-02-13T21:47:21.611Z 0 000-019 185 185 185 336 336 336 F T T <---
203582024-02-13T21:47:21.611Z 1 020-039 185 185 185 336 336 336 F T T <---
203592024-02-13T21:47:21.611Z 2 040-059 185 185 185 336 336 336 F T T <---
203602024-02-13T21:47:21.611Z 3 060-079 186 186 186 338 338 338 F T T <---
203612024-02-13T21:47:21.611Z 4 080-099 185 185 185 336 336 336 F F F
203622024-02-13T21:47:21.611Z 5 100-119 184 184 184 334 334 334 F T T <---
203632024-02-13T21:47:21.611Z 6 120-139 178 178 178 326 326 326 F F F
203642024-02-13T21:47:21.611Z 7 140-159 178 178 178 325 325 325 F T T <---
203652024-02-13T21:47:21.611Z 8 160-179 187 187 187 340 340 340 F F F
203662024-02-13T21:47:21.611Z 9 180-199 186 186 186 338 338 338 T T T
203672024-02-13T21:47:21.622Z 10 200-219 176 176 176 321 321 321 F T T <---
203682024-02-13T21:47:21.622Z 11 220-239 173 173 173 315 315 315 F T T <---
203692024-02-13T21:47:21.622Z 12 240-259 182 182 182 332 332 332 F T T <---
203702024-02-13T21:47:21.622Z 13 260-279 182 182 182 332 332 332 F T T <---
203712024-02-13T21:47:21.622Z 14 280-299 186 186 186 338 338 338 F F F
203722024-02-13T21:47:21.622Z 15 300-319 186 186 186 338 338 338 F F F
203732024-02-13T21:47:21.622Z 16 320-339 184 184 184 334 334 334 F F F
203742024-02-13T21:47:21.622Z 17 340-359 181 181 181 331 331 331 F T T <---
203752024-02-13T21:47:21.622Z 18 360-379 182 182 182 332 332 332 F F F
203762024-02-13T21:47:21.622Z 19 380-399 186 186 186 339 339 339 F T T <---
203772024-02-13T21:47:21.622Z 20 400-419 185 185 185 336 336 336 F T T <---
203782024-02-13T21:47:21.622Z 21 420-439 185 185 185 336 336 336 F T T <---
203792024-02-13T21:47:21.622Z 22 440-459 186 186 186 338 338 338 F F F
203802024-02-13T21:47:21.622Z 23 460-479 186 186 186 338 338 338 F T T <---
203812024-02-13T21:47:21.622Z 24 480-499 186 186 186 338 338 338 F T T <---
203822024-02-13T21:47:21.623Z 25 500-519 186 186 186 338 338 338 F F F
203832024-02-13T21:47:21.623Z 26 520-539 186 186 186 338 338 338 T T T
203842024-02-13T21:47:21.623Z 27 540-559 186 186 186 338 338 338 T T T
203852024-02-13T21:47:21.623Z 28 560-579 187 187 187 340 340 340 F T T <---
203862024-02-13T21:47:21.623Z 29 580-599 187 187 187 340 340 340 F F F
203872024-02-13T21:47:21.623ZMax gen: 187, Max flush: 340
203882024-02-13T21:47:21.623ZError: Difference in extent metadata found!
203892024-02-13T21:47:21.623ZA Difference in extent metadata is expected here
203902024-02-13T21:47:21.623ZOn loop 94
203912024-02-13T21:47:21.623Z
203922024-02-13T21:47:21.623ZVerifying data now
203932024-02-13T21:47:27.596ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
203942024-02-13T21:47:27.599Z543c543
203952024-02-13T21:47:27.599Z< 25,
203962024-02-13T21:47:27.599Z---
203972024-02-13T21:47:27.599Z> 26,
203982024-02-13T21:47:27.599Z552d551
203992024-02-13T21:47:27.599Z< 14,
204002024-02-13T21:47:27.599Z553a553
204012024-02-13T21:47:27.599Z> 15,
204022024-02-13T21:47:27.599Zdiff found after verify
204032024-02-13T21:47:27.602ZLoop: 94 Downstairs dump after verify (and repair):
204042024-02-13T21:47:27.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204052024-02-13T21:47:27.617Z 0 000-019 188 188 188 341 341 341 F F F
204062024-02-13T21:47:27.617Z 1 020-039 188 188 188 341 341 341 F F F
204072024-02-13T21:47:27.617Z 2 040-059 188 188 188 341 341 341 F F F
204082024-02-13T21:47:27.617Z 3 060-079 188 188 188 341 341 341 F F F
204092024-02-13T21:47:27.617Z 4 080-099 185 185 185 336 336 336 F F F
204102024-02-13T21:47:27.617Z 5 100-119 188 188 188 341 341 341 F F F
204112024-02-13T21:47:27.620Z 6 120-139 178 178 178 326 326 326 F F F
204122024-02-13T21:47:27.621Z 7 140-159 188 188 188 341 341 341 F F F
204132024-02-13T21:47:27.621Z 8 160-179 187 187 187 340 340 340 F F F
204142024-02-13T21:47:27.621Z 9 180-199 188 188 188 341 341 341 F F F
204152024-02-13T21:47:27.621Z 10 200-219 188 188 188 341 341 341 F F F
204162024-02-13T21:47:27.621Z 11 220-239 188 188 188 341 341 341 F F F
204172024-02-13T21:47:27.621Z 12 240-259 188 188 188 341 341 341 F F F
204182024-02-13T21:47:27.621Z 13 260-279 188 188 188 341 341 341 F F F
204192024-02-13T21:47:27.621Z 14 280-299 186 186 186 338 338 338 F F F
204202024-02-13T21:47:27.621Z 15 300-319 186 186 186 338 338 338 F F F
204212024-02-13T21:47:27.621Z 16 320-339 184 184 184 334 334 334 F F F
204222024-02-13T21:47:27.621Z 17 340-359 188 188 188 341 341 341 F F F
204232024-02-13T21:47:27.621Z 18 360-379 182 182 182 332 332 332 F F F
204242024-02-13T21:47:27.621Z 19 380-399 188 188 188 341 341 341 F F F
204252024-02-13T21:47:27.621Z 20 400-419 188 188 188 341 341 341 F F F
204262024-02-13T21:47:27.621Z 21 420-439 188 188 188 341 341 341 F F F
204272024-02-13T21:47:27.621Z 22 440-459 186 186 186 338 338 338 F F F
204282024-02-13T21:47:27.621Z 23 460-479 188 188 188 341 341 341 F F F
204292024-02-13T21:47:27.621Z 24 480-499 188 188 188 341 341 341 F F F
204302024-02-13T21:47:27.621Z 25 500-519 186 186 186 338 338 338 F F F
204312024-02-13T21:47:27.621Z 26 520-539 188 188 188 341 341 341 F F F
204322024-02-13T21:47:27.621Z 27 540-559 188 188 188 341 341 341 F F F
204332024-02-13T21:47:27.621Z 28 560-579 188 188 188 341 341 341 F F F
204342024-02-13T21:47:27.621Z 29 580-599 187 187 187 340 340 340 F F F
204352024-02-13T21:47:27.621ZMax gen: 188, Max flush: 341
204362024-02-13T21:47:27.622Z
204372024-02-13T21:47:27.622ZBegin loop 95
204382024-02-13T21:47:27.622ZDownstairs to restart: 1
204392024-02-13T21:47:27.639Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309087976Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901}
204402024-02-13T21:47:27.639Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309357738Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":Crucible runtime is spawned
204412024-02-13T21:47:27.639Z1901}
204422024-02-13T21:47:27.643Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.30940958Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901}
204432024-02-13T21:47:27.643Z{"msg":"Crucible 9af39587-8e7c-46e7-a443-43e97ecef609 has session id: a48fceac-1306-4ee5-880d-ba24d5ca178e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309449335Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204442024-02-13T21:47:27.643Z{"msg":"Upstairs opts: Upstairs UUID: 9af39587-8e7c-46e7-a443-43e97ecef609, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309503885Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204452024-02-13T21:47:27.643Z{"msg":"Crucible stats registered with UUID: 9af39587-8e7c-46e7-a443-43e97ecef609","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.30954483Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204462024-02-13T21:47:27.643Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.30957272Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901}
204472024-02-13T21:47:27.643Z{"msg":"9af39587-8e7c-46e7-a443-43e97ecef609 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309603489Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204482024-02-13T21:47:27.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309630449Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204492024-02-13T21:47:27.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309670775Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204502024-02-13T21:47:27.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309702193Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204512024-02-13T21:47:27.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309744358Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204522024-02-13T21:47:27.644Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.30979564Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204532024-02-13T21:47:27.644Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.30984972Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204542024-02-13T21:47:27.644Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309906229Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204552024-02-13T21:47:27.644Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309951863Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204562024-02-13T21:47:27.644Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.309981452Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204572024-02-13T21:47:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310008352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204582024-02-13T21:47:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310056155Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204592024-02-13T21:47:27.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.31009798Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"io task","client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204602024-02-13T21:47:27.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310679029Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204612024-02-13T21:47:27.644Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310739497Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204622024-02-13T21:47:27.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310775394Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204632024-02-13T21:47:27.644Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.310800215Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204642024-02-13T21:47:27.644Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311278691Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204652024-02-13T21:47:27.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311427747Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204662024-02-13T21:47:27.644Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311456937Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204672024-02-13T21:47:27.644Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311472661Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204682024-02-13T21:47:27.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311766354Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204692024-02-13T21:47:27.644Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311821934Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204702024-02-13T21:47:27.644Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.311854573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204712024-02-13T21:47:27.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.31202757Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204722024-02-13T21:47:27.644Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312067585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204732024-02-13T21:47:27.644Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312093916Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204742024-02-13T21:47:27.644Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312122865Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204752024-02-13T21:47:27.644Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312498419Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204762024-02-13T21:47:27.644Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312548101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204772024-02-13T21:47:27.645Z{"msg":"[0]R flush_numbers[0..12]: [341, 341, 341, 341, 336, 341, 326, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312573912Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204782024-02-13T21:47:27.645Z{"msg":"[0]R generation[0..12]: [188, 188, 188, 188, 185, 188, 178, 188, 187, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312608659Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204792024-02-13T21:47:27.645Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312635519Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204802024-02-13T21:47:27.648Z{"msg":"[1]R flush_numbers[0..12]: [341, 341, 341, 341, 336, 341, 326, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.31266078Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204812024-02-13T21:47:27.648Z{"msg":"[1]R generation[0..12]: [188, 188, 188, 188, 185, 188, 178, 188, 187, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312704474Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204822024-02-13T21:47:27.649Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.31274242Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
204832024-02-13T21:47:27.649Z,"pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204842024-02-13T21:47:27.649Z{"msg":"[2]R flush_numbers[0..12]: [341, 341, 341, 341, 336, 341, 326, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312783616Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204852024-02-13T21:47:27.649Z{"msg":"[2]R generation[0..12]: [188, 188, 188, 188, 185, 188, 178, 188, 187, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312820582Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204862024-02-13T21:47:27.649Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312860158Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204872024-02-13T21:47:27.649Z{"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312894985Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204882024-02-13T21:47:27.649Z{"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312931212Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204892024-02-13T21:47:27.649Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312958352Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204902024-02-13T21:47:27.649Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312972507Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"":"downstairs","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204912024-02-13T21:47:27.649Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.312998428Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204922024-02-13T21:47:27.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313011463Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"0","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204932024-02-13T21:47:27.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313027297Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"1","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204942024-02-13T21:47:27.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313058326Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"client":"2","":"downstairs","session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204952024-02-13T21:47:27.649Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313072591Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204962024-02-13T21:47:27.649Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313086536Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204972024-02-13T21:47:27.649Z{"msg":"9af39587-8e7c-46e7-a443-43e97ecef609 is now active with session: a48fceac-1306-4ee5-880d-ba24d5ca178e","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.31310115Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204982024-02-13T21:47:27.649Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313113436Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901,"session_id":"a48fceac-1306-4ee5-880d-ba24d5ca178e"}
204992024-02-13T21:47:27.649Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:26.313126621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901}
205002024-02-13T21:47:27.649ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205012024-02-13T21:47:27.649ZLoading write count information from file "/tmp/test_repair_verify.data"
205022024-02-13T21:47:27.649ZRun Repair workload
205032024-02-13T21:47:27.649Z01/30 Write block 222 len 16384 data: 21 22 20 18
205042024-02-13T21:47:27.649Z02/30 Write block 456 len 36864 data: 20 19 19 20 22 21 18 16 17
205052024-02-13T21:47:27.650Z03/30 Write block 495 len 36864 data: 16 15 16 15 13 14 10 12 8
205062024-02-13T21:47:27.650Z04/30 Write block 408 len 28672 data: 17 19 13 18 11 11 14
205072024-02-13T21:47:27.650Z05/30 Read block 66 len 32768
205082024-02-13T21:47:27.653Z06/30 Read block 491 len 36864
205092024-02-13T21:47:27.653Z07/30 Write block 528 len 20480 data: 15 14 13 16 17
205102024-02-13T21:47:27.653Z08/30 Flush
205112024-02-13T21:47:27.656Z09/30 Write block 582 len 16384 data: 10 12 13 14
205122024-02-13T21:47:27.656Z10/30 Write block 323 len 32768 data: 21 19 19 20 18 17 17 17
205132024-02-13T21:47:27.656Z11/30 Read block 243 len 8192
205142024-02-13T21:47:27.659Z12/30 Read block 168 len 28672
205152024-02-13T21:47:27.659Z13/30 Read block 224 len 40960
205162024-02-13T21:47:27.659Z14/30 Write block 413 len 8192 data: 12 15
205172024-02-13T21:47:27.659Z15/30 Read block 121 len 24576
205182024-02-13T21:47:27.662Z16/30 Read block 332 len 28672
205192024-02-13T21:47:27.662Z17/30 Read block 279 len 28672
205202024-02-13T21:47:27.662Z18/30 Write block 561 len 12288 data: 14 14 14
205212024-02-13T21:47:27.662Z19/30 Write block 70 len 8192 data: 19 17
205222024-02-13T21:47:27.665Z20/30 Write block 564 len 24576 data: 16 15 14 13 14 12
205232024-02-13T21:47:27.665Z21/30 Write block 439 len 4096 data: 10
205242024-02-13T21:47:27.665Z22/30 Read block 130 len 36864
205252024-02-13T21:47:27.665Z23/30 Write block 515 len 12288 data: 18 19 22
205262024-02-13T21:47:27.668Z24/30 Read block 567 len 32768
205272024-02-13T21:47:27.668Z25/30 Read block 332 len 8192
205282024-02-13T21:47:27.668Z26/30 Read block 102 len 12288
205292024-02-13T21:47:27.668Z27/30 Write block 554 len 36864 data: 17 17 18 17 14 10 12 15 15
205302024-02-13T21:47:27.671Z28/30 Write block 508 len 16384 data: 14 15 16 16
205312024-02-13T21:47:27.671Z29/30 Read block 512 len 4096
205322024-02-13T21:47:27.671Z30/30 Write block 78 len 8192 data: 8 8
205332024-02-13T21:47:27.671Z----------------------------------------------------------------
205342024-02-13T21:47:27.671Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
205352024-02-13T21:47:27.671ZStates: Active Active Active
205362024-02-13T21:47:27.671ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205372024-02-13T21:47:27.671Z 1 Acked 1000 Write 4 Done Done Done false
205382024-02-13T21:47:27.671Z 2 Acked 1001 Write 9 Done Done Done false
205392024-02-13T21:47:27.672Z 3 Acked 1002 Write 9 Done Done Done false
205402024-02-13T21:47:27.672Z 4 Acked 1003 Write 7 Done Done Done false
205412024-02-13T21:47:27.672Z 5 Acked 1004 Read 8 Done Done Done false
205422024-02-13T21:47:27.672Z 6 Acked 1005 Read 9 Done Sent Done false
205432024-02-13T21:47:27.672Z 7 Acked 1006 Write 5 Done Sent Done false
205442024-02-13T21:47:27.672Z 8 Acked 1007 Flush 0 Done Sent Done false
205452024-02-13T21:47:27.672Z 9 Acked 1008 Write 4 Done Sent Done false
205462024-02-13T21:47:27.672Z 10 Acked 1009 Write 8 Done Sent Done false
205472024-02-13T21:47:27.672Z 11 Acked 1010 Read 2 Done Sent Done false
205482024-02-13T21:47:27.672Z 12 Acked 1011 Read 7 Done Sent Done false
205492024-02-13T21:47:27.672Z 13 Acked 1012 Read 10 Done Sent Done false
205502024-02-13T21:47:27.672Z 14 Acked 1013 Write 2 Done Sent Done false
205512024-02-13T21:47:27.672Z 15 Acked 1014 Read 6 Done Sent Done false
205522024-02-13T21:47:27.672Z 16 Acked 1015 Read 7 Done Sent Done false
205532024-02-13T21:47:27.672Z 17 Acked 1016 Read 7 Done Sent Done false
205542024-02-13T21:47:27.672Z 18 Acked 1017 Write 3 Done Sent Done false
205552024-02-13T21:47:27.672Z 19 Acked 1018 Write 2 Done Sent Done false
205562024-02-13T21:47:27.672Z 20 Acked 1019 Write 6 Done Sent Done false
205572024-02-13T21:47:27.672Z 21 Acked 1020 Write 1 Done Sent Done false
205582024-02-13T21:47:27.672Z 22 Acked 1021 Read 9 Done Sent Done false
205592024-02-13T21:47:27.672Z 23 Acked 1022 Write 3 Done Sent Done false
205602024-02-13T21:47:27.672Z 24 Acked 1023 Read 8 Done Sent Done false
205612024-02-13T21:47:27.672Z 25 Acked 1024 Read 2 Done Sent Done false
205622024-02-13T21:47:27.672Z 26 Acked 1025 Read 3 Done Sent Done false
205632024-02-13T21:47:27.672Z 27 Acked 1026 Write 9 Done Sent Done false
205642024-02-13T21:47:27.673Z 28 Acked 1027 Write 4 Done Sent Done false
205652024-02-13T21:47:27.673Z 29 Acked 1028 Read 1 Done Sent Done false
205662024-02-13T21:47:27.673Z 30 Acked 1029 Write 2 Sent Sent Sent false
205672024-02-13T21:47:27.673Z STATES DS:0 DS:1 DS:2 TOTAL
205682024-02-13T21:47:27.673Z New 0 0 0 0
205692024-02-13T21:47:27.673Z Sent 1 25 1 27
205702024-02-13T21:47:27.673Z Done 29 5 29 63
205712024-02-13T21:47:27.673Z Skipped 0 0 0 0
205722024-02-13T21:47:27.673Z Error 0 0 0 0
205732024-02-13T21:47:27.673ZLast Flush: 1007 0 1007
205742024-02-13T21:47:27.673ZDownstairs last five completed:
205752024-02-13T21:47:27.673ZUpstairs last five completed: 30 29 28 27 26
205762024-02-13T21:47:27.673Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:26.341226228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1901}
205772024-02-13T21:47:27.673ZWrote out file "/tmp/test_repair_verify.data"
205782024-02-13T21:47:27.676Z
205792024-02-13T21:47:27.682ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205802024-02-13T21:47:27.696ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205812024-02-13T21:47:27.696Z 0 000-019 188 188 188 341 341 341 F F F
205822024-02-13T21:47:27.696Z 1 020-039 188 188 188 341 341 341 F F F
205832024-02-13T21:47:27.696Z 2 040-059 188 188 188 341 341 341 F F F
205842024-02-13T21:47:27.696Z 3 060-079 188 188 188 341 341 341 T F T <---
205852024-02-13T21:47:27.696Z 4 080-099 185 185 185 336 336 336 F F F
205862024-02-13T21:47:27.696Z 5 100-119 188 188 188 341 341 341 F F F
205872024-02-13T21:47:27.696Z 6 120-139 178 178 178 326 326 326 F F F
205882024-02-13T21:47:27.696Z 7 140-159 188 188 188 341 341 341 F F F
205892024-02-13T21:47:27.696Z 8 160-179 187 187 187 340 340 340 F F F
205902024-02-13T21:47:27.696Z 9 180-199 188 188 188 341 341 341 F F F
205912024-02-13T21:47:27.700Z 10 200-219 188 188 188 341 341 341 F F F
205922024-02-13T21:47:27.700Z 11 220-239 190 188 190 342 341 342 F T F <---
205932024-02-13T21:47:27.700Z 12 240-259 188 188 188 341 341 341 F F F
205942024-02-13T21:47:27.700Z 13 260-279 188 188 188 341 341 341 F F F
205952024-02-13T21:47:27.700Z 14 280-299 186 186 186 338 338 338 F F F
205962024-02-13T21:47:27.700Z 15 300-319 186 186 186 338 338 338 F F F
205972024-02-13T21:47:27.700Z 16 320-339 184 184 184 334 334 334 T F T <---
205982024-02-13T21:47:27.700Z 17 340-359 188 188 188 341 341 341 F F F
205992024-02-13T21:47:27.700Z 18 360-379 182 182 182 332 332 332 F F F
206002024-02-13T21:47:27.700Z 19 380-399 188 188 188 341 341 341 F F F
206012024-02-13T21:47:27.700Z 20 400-419 190 188 190 342 341 342 T T T <---
206022024-02-13T21:47:27.700Z 21 420-439 188 188 188 341 341 341 T F T <---
206032024-02-13T21:47:27.700Z 22 440-459 190 186 190 342 338 342 F T F <---
206042024-02-13T21:47:27.700Z 23 460-479 190 188 190 342 341 342 F T F <---
206052024-02-13T21:47:27.700Z 24 480-499 190 188 190 342 341 342 F T F <---
206062024-02-13T21:47:27.700Z 25 500-519 190 186 190 342 338 342 T T T <---
206072024-02-13T21:47:27.700Z 26 520-539 190 188 190 342 341 342 F F F <---
206082024-02-13T21:47:27.700Z 27 540-559 188 188 188 341 341 341 T F T <---
206092024-02-13T21:47:27.700Z 28 560-579 188 188 188 341 341 341 T F T <---
206102024-02-13T21:47:27.700Z 29 580-599 187 187 187 340 340 340 T F T <---
206112024-02-13T21:47:27.700ZMax gen: 190, Max flush: 342
206122024-02-13T21:47:27.700ZError: Difference in extent metadata found!
206132024-02-13T21:47:27.700ZA Difference in extent metadata is expected here
206142024-02-13T21:47:27.700ZOn loop 95
206152024-02-13T21:47:27.701Z
206162024-02-13T21:47:27.704ZVerifying data now
206172024-02-13T21:47:33.355ZNo change after verify
206182024-02-13T21:47:33.356ZLoop: 95 Downstairs dump after verify (and repair):
206192024-02-13T21:47:33.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206202024-02-13T21:47:33.371Z 0 000-019 188 188 188 341 341 341 F F F
206212024-02-13T21:47:33.371Z 1 020-039 188 188 188 341 341 341 F F F
206222024-02-13T21:47:33.371Z 2 040-059 188 188 188 341 341 341 F F F
206232024-02-13T21:47:33.371Z 3 060-079 191 191 191 343 343 343 F F F
206242024-02-13T21:47:33.371Z 4 080-099 185 185 185 336 336 336 F F F
206252024-02-13T21:47:33.371Z 5 100-119 188 188 188 341 341 341 F F F
206262024-02-13T21:47:33.371Z 6 120-139 178 178 178 326 326 326 F F F
206272024-02-13T21:47:33.371Z 7 140-159 188 188 188 341 341 341 F F F
206282024-02-13T21:47:33.371Z 8 160-179 187 187 187 340 340 340 F F F
206292024-02-13T21:47:33.371Z 9 180-199 188 188 188 341 341 341 F F F
206302024-02-13T21:47:33.372Z 10 200-219 188 188 188 341 341 341 F F F
206312024-02-13T21:47:33.372Z 11 220-239 190 190 190 342 342 342 F F F
206322024-02-13T21:47:33.372Z 12 240-259 188 188 188 341 341 341 F F F
206332024-02-13T21:47:33.372Z 13 260-279 188 188 188 341 341 341 F F F
206342024-02-13T21:47:33.372Z 14 280-299 186 186 186 338 338 338 F F F
206352024-02-13T21:47:33.372Z 15 300-319 186 186 186 338 338 338 F F F
206362024-02-13T21:47:33.375Z 16 320-339 191 191 191 343 343 343 F F F
206372024-02-13T21:47:33.375Z 17 340-359 188 188 188 341 341 341 F F F
206382024-02-13T21:47:33.375Z 18 360-379 182 182 182 332 332 332 F F F
206392024-02-13T21:47:33.375Z 19 380-399 188 188 188 341 341 341 F F F
206402024-02-13T21:47:33.375Z 20 400-419 191 191 191 343 343 343 F F F
206412024-02-13T21:47:33.375Z 21 420-439 191 191 191 343 343 343 F F F
206422024-02-13T21:47:33.375Z 22 440-459 190 190 190 342 342 342 F F F
206432024-02-13T21:47:33.375Z 23 460-479 190 190 190 342 342 342 F F F
206442024-02-13T21:47:33.375Z 24 480-499 190 190 190 342 342 342 F F F
206452024-02-13T21:47:33.375Z 25 500-519 191 191 191 343 343 343 F F F
206462024-02-13T21:47:33.375Z 26 520-539 190 190 190 342 342 342 F F F
206472024-02-13T21:47:33.376Z 27 540-559 191 191 191 343 343 343 F F F
206482024-02-13T21:47:33.376Z 28 560-579 191 191 191 343 343 343 F F F
206492024-02-13T21:47:33.376Z 29 580-599 191 191 191 343 343 343 F F F
206502024-02-13T21:47:33.376ZMax gen: 191, Max flush: 343
206512024-02-13T21:47:33.376Z
206522024-02-13T21:47:33.376ZBegin loop 96
206532024-02-13T21:47:33.376ZDownstairs to restart: 1
206542024-02-13T21:47:33.392Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.060463585Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
206552024-02-13T21:47:33.392Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.060702808Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
206562024-02-13T21:47:33.392Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.060719252Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
206572024-02-13T21:47:33.392Z{"msg":"Crucible bf54a490-ebdc-4c84-8a5c-3b30ce65eab7 has session id: 0af1d449-441e-4434-a86c-869f788beba8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.060743573Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206582024-02-13T21:47:33.397Z{"msg":"Upstairs opts: Upstairs UUID: bf54a490-ebdc-4c84-8a5c-3b30ce65eab7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.06078036Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206592024-02-13T21:47:33.397Z{"msg":"Crucible stats registered with UUID: bf54a490-ebdc-4c84-8a5c-3b30ce65eab7","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.060816197Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206602024-02-13T21:47:33.397ZCrucible runtime is spawned
206612024-02-13T21:47:33.397Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061008328Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
206622024-02-13T21:47:33.397Z{"msg":"bf54a490-ebdc-4c84-8a5c-3b30ce65eab7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061126655Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206632024-02-13T21:47:33.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061156024Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206642024-02-13T21:47:33.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061212583Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206652024-02-13T21:47:33.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.06130515Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206662024-02-13T21:47:33.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061351973Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206672024-02-13T21:47:33.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061390099Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206682024-02-13T21:47:33.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.0614161Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206692024-02-13T21:47:33.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061441111Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206702024-02-13T21:47:33.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.06147009Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206712024-02-13T21:47:33.397Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061495621Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206722024-02-13T21:47:33.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061521222Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206732024-02-13T21:47:33.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.061555789Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206742024-02-13T21:47:33.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.0615807Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"io task","client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206752024-02-13T21:47:33.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062207063Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206762024-02-13T21:47:33.397Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062255015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206772024-02-13T21:47:33.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062291832Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206782024-02-13T21:47:33.397Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062316533Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206792024-02-13T21:47:33.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.06273701Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206802024-02-13T21:47:33.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062842162Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206812024-02-13T21:47:33.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.062939037Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206822024-02-13T21:47:33.398Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063055625Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206832024-02-13T21:47:33.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063245656Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206842024-02-13T21:47:33.398Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063285921Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206852024-02-13T21:47:33.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063301546Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206862024-02-13T21:47:33.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063326617Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206872024-02-13T21:47:33.398Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063340631Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206882024-02-13T21:47:33.398Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063354087Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206892024-02-13T21:47:33.398Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.063608594Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206902024-02-13T21:47:33.398Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064018925Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206912024-02-13T21:47:33.398Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064053983Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206922024-02-13T21:47:33.404Z{"msg":"[0]R flush_numbers[0..12]: [341, 341, 341, 343, 336, 341, 326, 341, 340, 341, 341, 342]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064071517Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206932024-02-13T21:47:33.404Z{"msg":"[0]R generation[0..12]: [188, 188, 188, 191, 185, 188, 178, 188, 187, 188, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064098796Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206942024-02-13T21:47:33.404Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064113991Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206952024-02-13T21:47:33.404Z{"msg":"[1]R flush_numbers[0..12]: [341, 341, 341, 343, 336, 341, 326, 341, 340, 341, 341, 342]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064134963Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206962024-02-13T21:47:33.404Z{"msg":"[1]R generation[0..12]: [188, 188, 188, 191, 185, 188, 178, 188, 187, 188, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064149228Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206972024-02-13T21:47:33.405Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064169671Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206982024-02-13T21:47:33.405Z{"msg":"[2]R flush_numbers[0..12]: [341, 341, 341, 343, 336, 341, 326, 341, 340, 341, 341, 342]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064184325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
206992024-02-13T21:47:33.405Z{"msg":"[2]R generation[0..12]: [188, 188, 188, 191, 185, 188, 178, 188, 187, 188, 188, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064216694Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207002024-02-13T21:47:33.405Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064231638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207012024-02-13T21:47:33.405Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064252791Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
207022024-02-13T21:47:33.405Zdownstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207032024-02-13T21:47:33.405Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064269395Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207042024-02-13T21:47:33.405Z{"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064288618Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207052024-02-13T21:47:33.405Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064302093Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"":"downstairs","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207062024-02-13T21:47:33.405Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064322006Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207072024-02-13T21:47:33.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064334431Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"0","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207082024-02-13T21:47:33.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064353724Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"1","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207092024-02-13T21:47:33.405Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064367069Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"client":"2","":"downstairs","session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207102024-02-13T21:47:33.405Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064387252Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207112024-02-13T21:47:33.405Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064400237Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207122024-02-13T21:47:33.405Z{"msg":"bf54a490-ebdc-4c84-8a5c-3b30ce65eab7 is now active with session: 0af1d449-441e-4434-a86c-869f788beba8","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.06441955Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207132024-02-13T21:47:33.405Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064431876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909,"session_id":"0af1d449-441e-4434-a86c-869f788beba8"}
207142024-02-13T21:47:33.405Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:32.064450849Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
207152024-02-13T21:47:33.405ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207162024-02-13T21:47:33.405ZLoading write count information from file "/tmp/test_repair_verify.data"
207172024-02-13T21:47:33.405ZRun Repair workload
207182024-02-13T21:47:33.405Z01/30 Write block 361 len 28672 data: 11 12 15 14 15 14 17
207192024-02-13T21:47:33.405Z02/30 Write block 61 len 16384 data: 20 19 18 12
207202024-02-13T21:47:33.405Z03/30 Read block 232 len 36864
207212024-02-13T21:47:33.406Z04/30 Read block 181 len 4096
207222024-02-13T21:47:33.406Z05/30 Write block 235 len 36864 data: 12 9 9 14 14 15 15 16 14
207232024-02-13T21:47:33.409Z06/30 Read block 591 len 8192
207242024-02-13T21:47:33.409Z07/30 Read block 230 len 40960
207252024-02-13T21:47:33.409Z08/30 Read block 186 len 4096
207262024-02-13T21:47:33.409Z09/30 Read block 332 len 12288
207272024-02-13T21:47:33.409Z10/30 Write block 94 len 36864 data: 12 14 15 14 13 10 10 11 17
207282024-02-13T21:47:33.409Z11/30 Write block 216 len 20480 data: 19 21 21 20 21
207292024-02-13T21:47:33.409Z12/30 Write block 438 len 36864 data: 11 11 10 13 13 12 11 16 15
207302024-02-13T21:47:33.409Z13/30 Write block 7 len 28672 data: 13 12 14 15 18 18 18
207312024-02-13T21:47:33.409Z14/30 Write block 58 len 20480 data: 18 22 19 21 20
207322024-02-13T21:47:33.412Z15/30 Write block 354 len 36864 data: 17 18 18 17 16 14 14 12 13
207332024-02-13T21:47:33.412Z16/30 Write block 349 len 8192 data: 13 13
207342024-02-13T21:47:33.412Z17/30 Read block 589 len 28672
207352024-02-13T21:47:33.415Z18/30 Write block 473 len 40960 data: 20 21 20 18 15 15 12 12 14 16
207362024-02-13T21:47:33.415Z19/30 Write block 409 len 28672 data: 20 14 19 12 13 16 15
207372024-02-13T21:47:33.415Z20/30 Read block 258 len 36864
207382024-02-13T21:47:33.415Z21/30 Read block 100 len 24576
207392024-02-13T21:47:33.418Z22/30 Write block 495 len 36864 data: 17 16 17 16 14 15 11 13 9
207402024-02-13T21:47:33.418Z23/30 Write block 105 len 20480 data: 13 11 11 11 10
207412024-02-13T21:47:33.418Z24/30 Write block 45 len 8192 data: 12 14
207422024-02-13T21:47:33.418Z25/30 Read block 502 len 20480
207432024-02-13T21:47:33.421Z26/30 Write block 421 len 12288 data: 12 12 14
207442024-02-13T21:47:33.421Z27/30 Read block 560 len 24576
207452024-02-13T21:47:33.421Z28/30 Write block 37 len 12288 data: 13 15 14
207462024-02-13T21:47:33.421Z29/30 Read block 341 len 20480
207472024-02-13T21:47:33.425Z30/30 Read block 138 len 32768
207482024-02-13T21:47:33.425Z----------------------------------------------------------------
207492024-02-13T21:47:33.425Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207502024-02-13T21:47:33.425ZStates: Active Active Active
207512024-02-13T21:47:33.425ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207522024-02-13T21:47:33.425Z 1 Acked 1000 Write 7 Done Sent Done false
207532024-02-13T21:47:33.425Z 2 Acked 1001 Write 4 Done Sent Done false
207542024-02-13T21:47:33.425Z 3 Acked 1002 Read 9 Done Sent Done false
207552024-02-13T21:47:33.425Z 4 Acked 1003 Read 1 Done Sent Done false
207562024-02-13T21:47:33.425Z 5 Acked 1004 Write 9 Done Sent Done false
207572024-02-13T21:47:33.425Z 6 Acked 1005 Read 2 Done Sent Done false
207582024-02-13T21:47:33.425Z 7 Acked 1006 Read 10 Done Sent Done false
207592024-02-13T21:47:33.425Z 8 Acked 1007 Read 1 Done Sent Done false
207602024-02-13T21:47:33.425Z 9 Acked 1008 Read 3 Done Sent Done false
207612024-02-13T21:47:33.425Z 10 Acked 1009 Write 9 Done Sent Done false
207622024-02-13T21:47:33.425Z 11 Acked 1010 Write 5 Done Sent Done false
207632024-02-13T21:47:33.425Z 12 Acked 1011 Write 9 Done Sent Done false
207642024-02-13T21:47:33.425Z 13 Acked 1012 Write 7 Done Sent Done false
207652024-02-13T21:47:33.425Z 14 Acked 1013 Write 5 Done Sent Done false
207662024-02-13T21:47:33.426Z 15 Acked 1014 Write 9 Done Sent Done false
207672024-02-13T21:47:33.426Z 16 Acked 1015 Write 2 Done Sent Done false
207682024-02-13T21:47:33.426Z 17 Acked 1016 Read 7 Done Sent Done false
207692024-02-13T21:47:33.426Z 18 Acked 1017 Write 10 Done Sent Done false
207702024-02-13T21:47:33.426Z 19 Acked 1018 Write 7 Done Sent Done false
207712024-02-13T21:47:33.426Z 20 Acked 1019 Read 9 Done Sent Done false
207722024-02-13T21:47:33.426Z 21 Acked 1020 Read 6 Done Sent Done false
207732024-02-13T21:47:33.426Z 22 Acked 1021 Write 9 Done Sent Done false
207742024-02-13T21:47:33.426Z 23 Acked 1022 Write 5 Done Sent Done false
207752024-02-13T21:47:33.426Z 24 Acked 1023 Write 2 Done Sent Done false
207762024-02-13T21:47:33.426Z 25 Acked 1024 Read 5 Done Sent Done false
207772024-02-13T21:47:33.426Z 26 Acked 1025 Write 3 Done Sent Done false
207782024-02-13T21:47:33.426Z 27 Acked 1026 Read 6 Done Sent Done false
207792024-02-13T21:47:33.426Z 28 Acked 1027 Write 3 Done Sent Done false
207802024-02-13T21:47:33.426Z 29 Acked 1028 Read 5 Done Sent Done false
207812024-02-13T21:47:33.426Z 30 Acked 1029 Read 8 Done Sent Done false
207822024-02-13T21:47:33.426Z STATES DS:0 DS:1 DS:2 TOTAL
207832024-02-13T21:47:33.426Z New 0 0 0 0
207842024-02-13T21:47:33.426Z Sent 0 30 0 30
207852024-02-13T21:47:33.426Z Done 30 0 30 60
207862024-02-13T21:47:33.426Z Skipped 0 0 0 0
207872024-02-13T21:47:33.426Z Error 0 0 0 0
207882024-02-13T21:47:33.426ZLast Flush: 0 0 0
207892024-02-13T21:47:33.426ZDownstairs last five completed:
207902024-02-13T21:47:33.426ZUpstairs last five completed: 30 29 28 27 26
207912024-02-13T21:47:33.427Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:32.091837055Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1909}
207922024-02-13T21:47:33.427ZWrote out file "/tmp/test_repair_verify.data"
207932024-02-13T21:47:33.430Z
207942024-02-13T21:47:33.432ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207952024-02-13T21:47:33.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207962024-02-13T21:47:33.451Z 0 000-019 188 188 188 341 341 341 T F T <---
207972024-02-13T21:47:33.451Z 1 020-039 188 188 188 341 341 341 T F T <---
207982024-02-13T21:47:33.451Z 2 040-059 188 188 188 341 341 341 T F T <---
207992024-02-13T21:47:33.451Z 3 060-079 191 191 191 343 343 343 T F T <---
208002024-02-13T21:47:33.451Z 4 080-099 185 185 185 336 336 336 T F T <---
208012024-02-13T21:47:33.451Z 5 100-119 188 188 188 341 341 341 T F T <---
208022024-02-13T21:47:33.451Z 6 120-139 178 178 178 326 326 326 F F F
208032024-02-13T21:47:33.451Z 7 140-159 188 188 188 341 341 341 F F F
208042024-02-13T21:47:33.454Z 8 160-179 187 187 187 340 340 340 F F F
208052024-02-13T21:47:33.454Z 9 180-199 188 188 188 341 341 341 F F F
208062024-02-13T21:47:33.454Z 10 200-219 188 188 188 341 341 341 T F T <---
208072024-02-13T21:47:33.454Z 11 220-239 190 190 190 342 342 342 T F T <---
208082024-02-13T21:47:33.454Z 12 240-259 188 188 188 341 341 341 T F T <---
208092024-02-13T21:47:33.454Z 13 260-279 188 188 188 341 341 341 F F F
208102024-02-13T21:47:33.454Z 14 280-299 186 186 186 338 338 338 F F F
208112024-02-13T21:47:33.454Z 15 300-319 186 186 186 338 338 338 F F F
208122024-02-13T21:47:33.454Z 16 320-339 191 191 191 343 343 343 F F F
208132024-02-13T21:47:33.454Z 17 340-359 188 188 188 341 341 341 T F T <---
208142024-02-13T21:47:33.454Z 18 360-379 182 182 182 332 332 332 T F T <---
208152024-02-13T21:47:33.455Z 19 380-399 188 188 188 341 341 341 F F F
208162024-02-13T21:47:33.455Z 20 400-419 191 191 191 343 343 343 T F T <---
208172024-02-13T21:47:33.455Z 21 420-439 191 191 191 343 343 343 T F T <---
208182024-02-13T21:47:33.455Z 22 440-459 190 190 190 342 342 342 T F T <---
208192024-02-13T21:47:33.455Z 23 460-479 190 190 190 342 342 342 T F T <---
208202024-02-13T21:47:33.455Z 24 480-499 190 190 190 342 342 342 T F T <---
208212024-02-13T21:47:33.455Z 25 500-519 191 191 191 343 343 343 T F T <---
208222024-02-13T21:47:33.455Z 26 520-539 190 190 190 342 342 342 F F F
208232024-02-13T21:47:33.455Z 27 540-559 191 191 191 343 343 343 F F F
208242024-02-13T21:47:33.455Z 28 560-579 191 191 191 343 343 343 F F F
208252024-02-13T21:47:33.455Z 29 580-599 191 191 191 343 343 343 F F F
208262024-02-13T21:47:33.455ZMax gen: 191, Max flush: 343
208272024-02-13T21:47:33.455ZError: Difference in extent metadata found!
208282024-02-13T21:47:33.455ZA Difference in extent metadata is expected here
208292024-02-13T21:47:33.455ZOn loop 96
208302024-02-13T21:47:33.455Z
208312024-02-13T21:47:33.458ZVerifying data now
208322024-02-13T21:47:38.293ZNo change after verify
208332024-02-13T21:47:38.293ZLoop: 96 Downstairs dump after verify (and repair):
208342024-02-13T21:47:38.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208352024-02-13T21:47:38.307Z 0 000-019 192 192 192 344 344 344 F F F
208362024-02-13T21:47:38.307Z 1 020-039 192 192 192 344 344 344 F F F
208372024-02-13T21:47:38.307Z 2 040-059 192 192 192 344 344 344 F F F
208382024-02-13T21:47:38.307Z 3 060-079 192 192 192 344 344 344 F F F
208392024-02-13T21:47:38.307Z 4 080-099 192 192 192 344 344 344 F F F
208402024-02-13T21:47:38.307Z 5 100-119 192 192 192 344 344 344 F F F
208412024-02-13T21:47:38.307Z 6 120-139 178 178 178 326 326 326 F F F
208422024-02-13T21:47:38.308Z 7 140-159 188 188 188 341 341 341 F F F
208432024-02-13T21:47:38.308Z 8 160-179 187 187 187 340 340 340 F F F
208442024-02-13T21:47:38.308Z 9 180-199 188 188 188 341 341 341 F F F
208452024-02-13T21:47:38.308Z 10 200-219 192 192 192 344 344 344 F F F
208462024-02-13T21:47:38.308Z 11 220-239 192 192 192 344 344 344 F F F
208472024-02-13T21:47:38.308Z 12 240-259 192 192 192 344 344 344 F F F
208482024-02-13T21:47:38.311Z 13 260-279 188 188 188 341 341 341 F F F
208492024-02-13T21:47:38.311Z 14 280-299 186 186 186 338 338 338 F F F
208502024-02-13T21:47:38.311Z 15 300-319 186 186 186 338 338 338 F F F
208512024-02-13T21:47:38.312Z 16 320-339 191 191 191 343 343 343 F F F
208522024-02-13T21:47:38.312Z 17 340-359 192 192 192 344 344 344 F F F
208532024-02-13T21:47:38.312Z 18 360-379 192 192 192 344 344 344 F F F
208542024-02-13T21:47:38.312Z 19 380-399 188 188 188 341 341 341 F F F
208552024-02-13T21:47:38.312Z 20 400-419 192 192 192 344 344 344 F F F
208562024-02-13T21:47:38.312Z 21 420-439 192 192 192 344 344 344 F F F
208572024-02-13T21:47:38.312Z 22 440-459 192 192 192 344 344 344 F F F
208582024-02-13T21:47:38.312Z 23 460-479 192 192 192 344 344 344 F F F
208592024-02-13T21:47:38.312Z 24 480-499 192 192 192 344 344 344 F F F
208602024-02-13T21:47:38.312Z 25 500-519 192 192 192 344 344 344 F F F
208612024-02-13T21:47:38.312Z 26 520-539 190 190 190 342 342 342 F F F
208622024-02-13T21:47:38.312Z 27 540-559 191 191 191 343 343 343 F F F
208632024-02-13T21:47:38.312Z 28 560-579 191 191 191 343 343 343 F F F
208642024-02-13T21:47:38.312Z 29 580-599 191 191 191 343 343 343 F F F
208652024-02-13T21:47:38.312ZMax gen: 192, Max flush: 344
208662024-02-13T21:47:38.312Z
208672024-02-13T21:47:38.312ZBegin loop 97
208682024-02-13T21:47:38.312ZDownstairs to restart: 2
208692024-02-13T21:47:38.328Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.994751061Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
208702024-02-13T21:47:38.328Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.994998052Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
208712024-02-13T21:47:38.328Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995015116Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
208722024-02-13T21:47:38.332Z{"msg":"Crucible 17dde751-7a65-4945-8c29-b755b1741e4d has session id: d671999b-edd0-4dfe-8526-fc591a57408d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995035948Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208732024-02-13T21:47:38.333Z{"msg":"Upstairs opts: Upstairs UUID: 17dde751-7a65-4945-8c29-b755b1741e4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"nameCrucible runtime is spawned
208742024-02-13T21:47:38.333Z":"crucible","level":30,"time":"2024-02-13T21:47:36.995092867Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208752024-02-13T21:47:38.333Z{"msg":"Crucible stats registered with UUID: 17dde751-7a65-4945-8c29-b755b1741e4d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.99514142Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208762024-02-13T21:47:38.333Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995164801Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
208772024-02-13T21:47:38.333Z{"msg":"17dde751-7a65-4945-8c29-b755b1741e4d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995297433Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208782024-02-13T21:47:38.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995341307Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208792024-02-13T21:47:38.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995371506Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208802024-02-13T21:47:38.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.99541657Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208812024-02-13T21:47:38.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.9954425Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208822024-02-13T21:47:38.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995465932Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208832024-02-13T21:47:38.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995491303Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208842024-02-13T21:47:38.333Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995534587Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208852024-02-13T21:47:38.333Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995569085Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208862024-02-13T21:47:38.333Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995598184Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208872024-02-13T21:47:38.333Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995634191Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208882024-02-13T21:47:38.333Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995660891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208892024-02-13T21:47:38.333Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.995698168Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"io task","client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208902024-02-13T21:47:38.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.996331348Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208912024-02-13T21:47:38.333Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.996368255Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208922024-02-13T21:47:38.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.996385259Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208932024-02-13T21:47:38.333Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.996420306Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208942024-02-13T21:47:38.333Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.996936449Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208952024-02-13T21:47:38.333Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997046509Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208962024-02-13T21:47:38.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997410906Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208972024-02-13T21:47:38.334Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997488549Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208982024-02-13T21:47:38.334Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997524475Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
208992024-02-13T21:47:38.334Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.99753875Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209002024-02-13T21:47:38.334Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997553325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209012024-02-13T21:47:38.334Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997575537Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209022024-02-13T21:47:38.334Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997588172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209032024-02-13T21:47:38.334Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997602337Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209042024-02-13T21:47:38.334Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.997998623Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209052024-02-13T21:47:38.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998404086Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209062024-02-13T21:47:38.340Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998455838Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209072024-02-13T21:47:38.340Z{"msg":"[0]R flush_numbers[0..12]: [344, 344, 344, 344, 344, 344, 326, 341, 340, 341, 344, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998485837Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209082024-02-13T21:47:38.340Z{"msg":"[0]R generation[0..12]: [192, 192, 192, 192, 192, 192, 178, 188, 187, 188, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998511817Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209092024-02-13T21:47:38.341Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.9985311Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209102024-02-13T21:47:38.341Z{"msg":"[1]R flush_numbers[0..12]: [344, 344, 344, 344, 344, 344, 326, 341, 340, 341, 344, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998557071Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209112024-02-13T21:47:38.341Z{"msg":"[1]R generation[0..12]: [192, 192, 192, 192, 192, 192, 178, 188, 187, 188, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998575834Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
209122024-02-13T21:47:38.341Zd671999b-edd0-4dfe-8526-fc591a57408d"}
209132024-02-13T21:47:38.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998605403Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209142024-02-13T21:47:38.341Z{"msg":"[2]R flush_numbers[0..12]: [344, 344, 344, 344, 344, 344, 326, 341, 340, 341, 344, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998631864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209152024-02-13T21:47:38.341Z{"msg":"[2]R generation[0..12]: [192, 192, 192, 192, 192, 192, 178, 188, 187, 188, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998658854Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209162024-02-13T21:47:38.341Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998677477Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209172024-02-13T21:47:38.341Z{"msg":"Max found gen is 193","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998702238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209182024-02-13T21:47:38.341Z{"msg":"Generation requested: 194 >= found:193","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998722891Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209192024-02-13T21:47:38.341Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.99875289Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209202024-02-13T21:47:38.341Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998770144Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"":"downstairs","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209212024-02-13T21:47:38.341Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998795015Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209222024-02-13T21:47:38.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998822175Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"0","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209232024-02-13T21:47:38.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998846366Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"1","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209242024-02-13T21:47:38.341Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998888761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"client":"2","":"downstairs","session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209252024-02-13T21:47:38.341Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998925997Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209262024-02-13T21:47:38.341Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998949978Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209272024-02-13T21:47:38.341Z{"msg":"17dde751-7a65-4945-8c29-b755b1741e4d is now active with session: d671999b-edd0-4dfe-8526-fc591a57408d","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.998988344Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
209282024-02-13T21:47:38.341Z,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209292024-02-13T21:47:38.341Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.99902933Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917,"session_id":"d671999b-edd0-4dfe-8526-fc591a57408d"}
209302024-02-13T21:47:38.341Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:36.999052901Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
209312024-02-13T21:47:38.341ZLoading write count information from file "/tmp/test_repair_verify.data"
209322024-02-13T21:47:38.341ZRun Repair workload
209332024-02-13T21:47:38.342Z01/30 Write block 512 len 24576 data: 15 15 19 19 20 23
209342024-02-13T21:47:38.342Z02/30 Read block 256 len 36864
209352024-02-13T21:47:38.342Z03/30 Read block 571 len 32768
209362024-02-13T21:47:38.342Z04/30 Read block 138 len 32768
209372024-02-13T21:47:38.345Z05/30 Write block 575 len 8192 data: 7 9
209382024-02-13T21:47:38.345Z06/30 Read block 460 len 20480
209392024-02-13T21:47:38.345Z07/30 Read block 200 len 28672
209402024-02-13T21:47:38.345Z08/30 Write block 34 len 36864 data: 18 20 17 14 16 15 16 15 13
209412024-02-13T21:47:38.345Z09/30 Read block 7 len 28672
209422024-02-13T21:47:38.345Z10/30 Read block 398 len 8192
209432024-02-13T21:47:38.345Z11/30 Read block 35 len 28672
209442024-02-13T21:47:38.345Z12/30 Read block 72 len 8192
209452024-02-13T21:47:38.345Z13/30 Write block 364 len 20480 data: 15 16 15 18 16
209462024-02-13T21:47:38.346Z14/30 Write block 94 len 12288 data: 13 15 16
209472024-02-13T21:47:38.348Z15/30 Read block 81 len 32768
209482024-02-13T21:47:38.348Z16/30 Read block 406 len 32768
209492024-02-13T21:47:38.348Z17/30 Write block 537 len 40960 data: 17 20 23 26 21 21 20 15 17 15
209502024-02-13T21:47:38.348Z18/30 Flush
209512024-02-13T21:47:38.352Z19/30 Write block 392 len 24576 data: 12 13 14 12 11 11
209522024-02-13T21:47:38.355Z20/30 Write block 260 len 40960 data: 8 9 10 11 12 11 12 10 13 13
209532024-02-13T21:47:38.355Z21/30 Flush
209542024-02-13T21:47:38.358Z22/30 Write block 517 len 40960 data: 24 22 25 24 22 20 20 15 12 12
209552024-02-13T21:47:38.358Z23/30 Read block 278 len 32768
209562024-02-13T21:47:38.361Z24/30 Write block 34 len 12288 data: 19 21 18
209572024-02-13T21:47:38.361Z25/30 Write block 145 len 20480 data: 18 14 14 12 13
209582024-02-13T21:47:38.361Z26/30 Read block 502 len 40960
209592024-02-13T21:47:38.365Z27/30 Read block 92 len 32768
209602024-02-13T21:47:38.365Z28/30 Read block 416 len 16384
209612024-02-13T21:47:38.365Z29/30 Write block 264 len 16384 data: 13 12 13 11
209622024-02-13T21:47:38.365Z30/30 Write block 205 len 32768 data: 11 6 7 10 11 11 11 13
209632024-02-13T21:47:38.365Z----------------------------------------------------------------
209642024-02-13T21:47:38.365Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209652024-02-13T21:47:38.365ZStates: Active Active Active
209662024-02-13T21:47:38.365ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209672024-02-13T21:47:38.365Z 1 Acked 1000 Write 6 Done Done Sent false
209682024-02-13T21:47:38.365Z 2 Acked 1001 Read 9 Done Done Sent false
209692024-02-13T21:47:38.366Z 3 Acked 1002 Read 8 Done Done Sent false
209702024-02-13T21:47:38.366Z 4 Acked 1003 Read 8 Done Done Sent false
209712024-02-13T21:47:38.366Z 5 Acked 1004 Write 2 Done Done Sent false
209722024-02-13T21:47:38.366Z 6 Acked 1005 Read 5 Done Done Sent false
209732024-02-13T21:47:38.366Z 7 Acked 1006 Read 7 Done Done Sent false
209742024-02-13T21:47:38.366Z 8 Acked 1007 Write 9 Done Done Sent false
209752024-02-13T21:47:38.366Z 9 Acked 1008 Read 7 Done Done Sent false
209762024-02-13T21:47:38.366Z 10 Acked 1009 Read 2 Done Done Sent false
209772024-02-13T21:47:38.366Z 11 Acked 1010 Read 7 Done Done Sent false
209782024-02-13T21:47:38.366Z 12 Acked 1011 Read 2 Done Done Sent false
209792024-02-13T21:47:38.366Z 13 Acked 1012 Write 5 Done Done Sent false
209802024-02-13T21:47:38.366Z 14 Acked 1013 Write 3 Done Done Sent false
209812024-02-13T21:47:38.366Z 15 Acked 1014 Read 8 Done Done Sent false
209822024-02-13T21:47:38.366Z 16 Acked 1015 Read 8 Done Done Sent false
209832024-02-13T21:47:38.367Z 17 Acked 1016 Write 10 Done Done Sent false
209842024-02-13T21:47:38.367Z 18 Acked 1017 Flush 0 Done Done Sent false
209852024-02-13T21:47:38.367Z 19 Acked 1018 Write 6 Done Done Sent false
209862024-02-13T21:47:38.367Z 20 Acked 1019 Write 10 Done Done Sent false
209872024-02-13T21:47:38.367Z 21 Acked 1020 Flush 0 Done Done Sent false
209882024-02-13T21:47:38.367Z 22 Acked 1021 Write 10 Done Done Sent false
209892024-02-13T21:47:38.367Z 23 Acked 1022 Read 8 Done Done Sent false
209902024-02-13T21:47:38.367Z 24 Acked 1023 Write 3 Done Done Sent false
209912024-02-13T21:47:38.367Z 25 Acked 1024 Write 5 Done Done Sent false
209922024-02-13T21:47:38.367Z 26 Acked 1025 Read 10 Done Done Sent false
209932024-02-13T21:47:38.367Z 27 Acked 1026 Read 8 Done Done Sent false
209942024-02-13T21:47:38.367Z 28 Acked 1027 Read 4 Done Done Sent false
209952024-02-13T21:47:38.368Z 29 Acked 1028 Write 4 Done Done Sent false
209962024-02-13T21:47:38.368Z 30 Acked 1029 Write 8 Sent Sent Sent false
209972024-02-13T21:47:38.368Z STATES DS:0 DS:1 DS:2 TOTAL
209982024-02-13T21:47:38.368Z New 0 0 0 0
209992024-02-13T21:47:38.368Z Sent 1 1 30 32
210002024-02-13T21:47:38.368Z Done 29 29 0 58
210012024-02-13T21:47:38.368Z Skipped 0 0 0 0
210022024-02-13T21:47:38.368Z Error 0 0 0 0
210032024-02-13T21:47:38.368ZLast Flush: 1020 1020 0
210042024-02-13T21:47:38.368ZDownstairs last five completed:
210052024-02-13T21:47:38.368ZUpstairs last five completed: 30 29 28 27 26
210062024-02-13T21:47:38.373Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:37.031314109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1917}
210072024-02-13T21:47:38.374ZWrote out file "/tmp/test_repair_verify.data"
210082024-02-13T21:47:38.374Z
210092024-02-13T21:47:38.377ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
210102024-02-13T21:47:38.389ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210112024-02-13T21:47:38.390Z 0 000-019 192 192 192 344 344 344 F F F
210122024-02-13T21:47:38.390Z 1 020-039 194 194 192 345 345 344 T T F <---
210132024-02-13T21:47:38.390Z 2 040-059 194 194 192 345 345 344 F F F <---
210142024-02-13T21:47:38.390Z 3 060-079 192 192 192 344 344 344 F F F
210152024-02-13T21:47:38.390Z 4 080-099 194 194 192 345 345 344 F F F <---
210162024-02-13T21:47:38.390Z 5 100-119 192 192 192 344 344 344 F F F
210172024-02-13T21:47:38.390Z 6 120-139 178 178 178 326 326 326 F F F
210182024-02-13T21:47:38.390Z 7 140-159 188 188 188 341 341 341 T T F <---
210192024-02-13T21:47:38.390Z 8 160-179 187 187 187 340 340 340 F F F
210202024-02-13T21:47:38.390Z 9 180-199 188 188 188 341 341 341 F F F
210212024-02-13T21:47:38.390Z 10 200-219 192 192 192 344 344 344 T T F <---
210222024-02-13T21:47:38.390Z 11 220-239 192 192 192 344 344 344 F F F
210232024-02-13T21:47:38.390Z 12 240-259 192 192 192 344 344 344 F F F
210242024-02-13T21:47:38.390Z 13 260-279 194 194 188 346 346 341 T T F <---
210252024-02-13T21:47:38.394Z 14 280-299 186 186 186 338 338 338 F F F
210262024-02-13T21:47:38.394Z 15 300-319 186 186 186 338 338 338 F F F
210272024-02-13T21:47:38.394Z 16 320-339 191 191 191 343 343 343 F F F
210282024-02-13T21:47:38.394Z 17 340-359 192 192 192 344 344 344 F F F
210292024-02-13T21:47:38.394Z 18 360-379 194 194 192 345 345 344 F F F <---
210302024-02-13T21:47:38.394Z 19 380-399 194 194 188 346 346 341 F F F <---
210312024-02-13T21:47:38.394Z 20 400-419 192 192 192 344 344 344 F F F
210322024-02-13T21:47:38.394Z 21 420-439 192 192 192 344 344 344 F F F
210332024-02-13T21:47:38.394Z 22 440-459 192 192 192 344 344 344 F F F
210342024-02-13T21:47:38.394Z 23 460-479 192 192 192 344 344 344 F F F
210352024-02-13T21:47:38.394Z 24 480-499 192 192 192 344 344 344 F F F
210362024-02-13T21:47:38.394Z 25 500-519 194 194 192 345 345 344 T T F <---
210372024-02-13T21:47:38.394Z 26 520-539 194 194 190 345 345 342 T T F <---
210382024-02-13T21:47:38.395Z 27 540-559 194 194 191 345 345 343 F F F <---
210392024-02-13T21:47:38.395Z 28 560-579 194 194 191 345 345 343 F F F <---
210402024-02-13T21:47:38.395Z 29 580-599 191 191 191 343 343 343 F F F
210412024-02-13T21:47:38.395ZMax gen: 194, Max flush: 346
210422024-02-13T21:47:38.395ZError: Difference in extent metadata found!
210432024-02-13T21:47:38.395ZA Difference in extent metadata is expected here
210442024-02-13T21:47:38.395ZOn loop 97
210452024-02-13T21:47:38.395Z
210462024-02-13T21:47:38.398ZVerifying data now
210472024-02-13T21:47:42.991ZNo change after verify
210482024-02-13T21:47:42.994ZLoop: 97 Downstairs dump after verify (and repair):
210492024-02-13T21:47:43.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210502024-02-13T21:47:43.006Z 0 000-019 192 192 192 344 344 344 F F F
210512024-02-13T21:47:43.006Z 1 020-039 195 195 195 347 347 347 F F F
210522024-02-13T21:47:43.006Z 2 040-059 194 194 194 345 345 345 F F F
210532024-02-13T21:47:43.006Z 3 060-079 192 192 192 344 344 344 F F F
210542024-02-13T21:47:43.006Z 4 080-099 194 194 194 345 345 345 F F F
210552024-02-13T21:47:43.006Z 5 100-119 192 192 192 344 344 344 F F F
210562024-02-13T21:47:43.006Z 6 120-139 178 178 178 326 326 326 F F F
210572024-02-13T21:47:43.006Z 7 140-159 195 195 195 347 347 347 F F F
210582024-02-13T21:47:43.006Z 8 160-179 187 187 187 340 340 340 F F F
210592024-02-13T21:47:43.006Z 9 180-199 188 188 188 341 341 341 F F F
210602024-02-13T21:47:43.006Z 10 200-219 195 195 195 347 347 347 F F F
210612024-02-13T21:47:43.006Z 11 220-239 192 192 192 344 344 344 F F F
210622024-02-13T21:47:43.006Z 12 240-259 192 192 192 344 344 344 F F F
210632024-02-13T21:47:43.010Z 13 260-279 195 195 195 347 347 347 F F F
210642024-02-13T21:47:43.010Z 14 280-299 186 186 186 338 338 338 F F F
210652024-02-13T21:47:43.010Z 15 300-319 186 186 186 338 338 338 F F F
210662024-02-13T21:47:43.010Z 16 320-339 191 191 191 343 343 343 F F F
210672024-02-13T21:47:43.010Z 17 340-359 192 192 192 344 344 344 F F F
210682024-02-13T21:47:43.010Z 18 360-379 194 194 194 345 345 345 F F F
210692024-02-13T21:47:43.010Z 19 380-399 194 194 194 346 346 346 F F F
210702024-02-13T21:47:43.010Z 20 400-419 192 192 192 344 344 344 F F F
210712024-02-13T21:47:43.010Z 21 420-439 192 192 192 344 344 344 F F F
210722024-02-13T21:47:43.010Z 22 440-459 192 192 192 344 344 344 F F F
210732024-02-13T21:47:43.010Z 23 460-479 192 192 192 344 344 344 F F F
210742024-02-13T21:47:43.010Z 24 480-499 192 192 192 344 344 344 F F F
210752024-02-13T21:47:43.010Z 25 500-519 195 195 195 347 347 347 F F F
210762024-02-13T21:47:43.010Z 26 520-539 195 195 195 347 347 347 F F F
210772024-02-13T21:47:43.010Z 27 540-559 194 194 194 345 345 345 F F F
210782024-02-13T21:47:43.010Z 28 560-579 194 194 194 345 345 345 F F F
210792024-02-13T21:47:43.010Z 29 580-599 191 191 191 343 343 343 F F F
210802024-02-13T21:47:43.010ZMax gen: 195, Max flush: 347
210812024-02-13T21:47:43.011Z
210822024-02-13T21:47:43.011ZBegin loop 98
210832024-02-13T21:47:43.011ZDownstairs to restart: 1
210842024-02-13T21:47:43.028Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693367883Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
210852024-02-13T21:47:43.033Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693638285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
210862024-02-13T21:47:43.033ZCrucible runtime is spawned
210872024-02-13T21:47:43.033Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693678531Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
210882024-02-13T21:47:43.033Z{"msg":"Crucible 1e1daab6-72e8-4195-81d8-5736288c5442 has session id: 5280a4e6-9e4f-49b8-8600-8a1d6e6b1550","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69370734Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210892024-02-13T21:47:43.033Z{"msg":"Upstairs opts: Upstairs UUID: 1e1daab6-72e8-4195-81d8-5736288c5442, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693754343Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210902024-02-13T21:47:43.033Z{"msg":"Crucible stats registered with UUID: 1e1daab6-72e8-4195-81d8-5736288c5442","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693798517Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210912024-02-13T21:47:43.033Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69381665Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
210922024-02-13T21:47:43.033Z{"msg":"1e1daab6-72e8-4195-81d8-5736288c5442 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693853437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210932024-02-13T21:47:43.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693870151Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210942024-02-13T21:47:43.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693894072Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210952024-02-13T21:47:43.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693933068Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210962024-02-13T21:47:43.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.693989448Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210972024-02-13T21:47:43.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694024325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210982024-02-13T21:47:43.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694060462Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
210992024-02-13T21:47:43.033Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69409429Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211002024-02-13T21:47:43.033Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69414679Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211012024-02-13T21:47:43.033Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694191074Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211022024-02-13T21:47:43.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694233869Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211032024-02-13T21:47:43.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694264248Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211042024-02-13T21:47:43.034Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694302764Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"io task","client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211052024-02-13T21:47:43.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694913003Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211062024-02-13T21:47:43.034Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69494719Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211072024-02-13T21:47:43.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694962505Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211082024-02-13T21:47:43.034Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.694995013Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211092024-02-13T21:47:43.034Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.695546183Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211102024-02-13T21:47:43.034Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.695679325Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211112024-02-13T21:47:43.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.695817664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211122024-02-13T21:47:43.034Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69585639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211132024-02-13T21:47:43.034Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696090785Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211142024-02-13T21:47:43.034Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696130721Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211152024-02-13T21:47:43.034Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696150214Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211162024-02-13T21:47:43.034Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696252077Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211172024-02-13T21:47:43.034Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69629909Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211182024-02-13T21:47:43.034Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69632545Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211192024-02-13T21:47:43.034Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69635341Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211202024-02-13T21:47:43.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696802887Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211212024-02-13T21:47:43.038Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696844202Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211222024-02-13T21:47:43.038Z{"msg":"[0]R flush_numbers[0..12]: [344, 347, 345, 344, 345, 344, 326, 347, 340, 341, 347, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696860846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211232024-02-13T21:47:43.038Z{"msg":"[0]R generation[0..12]: [192, 195, 194, 192, 194, 192, 178, 195, 187, 188, 195, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696883238Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211242024-02-13T21:47:43.038Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696898172Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211252024-02-13T21:47:43.038Z{"msg":"[1]R flush_numbers[0..12]: [344, 347, 345, 344, 345, 344, 326, 347, 340, 341, 347, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696919945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211262024-02-13T21:47:43.039Z{"msg":"[1]R generation[0..12]: [192, 195, 194, 192, 194, 192, 178, 195, 187, 188, 195, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696934369Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211272024-02-13T21:47:43.039Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696955372Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211282024-02-13T21:47:43.039Z{Wait for a query_work_queue command to finish before sending IO
211292024-02-13T21:47:43.039Z"msg":"[2]R flush_numbers[0..12]: [344, 347, 345, 344, 345, 344, 326, 347, 340, 341, 347, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696972525Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211302024-02-13T21:47:43.039Z{"msg":"[2]R generation[0..12]: [192, 195, 194, 192, 194, 192, 178, 195, 187, 188, 195, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.696993928Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211312024-02-13T21:47:43.039Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697008392Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211322024-02-13T21:47:43.039Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697028805Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211332024-02-13T21:47:43.039Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69704266Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211342024-02-13T21:47:43.039Z{"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697062513Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211352024-02-13T21:47:43.039Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697076638Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"":"downstairs","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211362024-02-13T21:47:43.039Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.69709716Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211372024-02-13T21:47:43.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697109566Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"0","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211382024-02-13T21:47:43.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697129459Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"1","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211392024-02-13T21:47:43.039Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697142814Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"client":"2","":"downstairs","session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211402024-02-13T21:47:43.039Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697162437Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211412024-02-13T21:47:43.039Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697174732Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211422024-02-13T21:47:43.039Z{"msg":"1e1daab6-72e8-4195-81d8-5736288c5442 is now active with session: 5280a4e6-9e4f-49b8-8600-8a1d6e6b1550","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697187647Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211432024-02-13T21:47:43.039Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697199603Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925,"session_id":"5280a4e6-9e4f-49b8-8600-8a1d6e6b1550"}
211442024-02-13T21:47:43.039Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:41.697212129Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
211452024-02-13T21:47:43.039ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211462024-02-13T21:47:43.039ZLoading write count information from file "/tmp/test_repair_verify.data"
211472024-02-13T21:47:43.039ZRun Repair workload
211482024-02-13T21:47:43.039Z01/30 Write block 549 len 36864 data: 15 16 22 19 18 18 18 19 18
211492024-02-13T21:47:43.040Z02/30 Read block 354 len 24576
211502024-02-13T21:47:43.040Z03/30 Write block 208 len 12288 data: 11 12 12
211512024-02-13T21:47:43.040Z04/30 Read block 476 len 36864
211522024-02-13T21:47:43.043Z05/30 Write block 406 len 24576 data: 15 14 18 21 15 20
211532024-02-13T21:47:43.043Z06/30 Read block 199 len 16384
211542024-02-13T21:47:43.043Z07/30 Write block 390 len 40960 data: 13 12 13 14 15 13 12 12 12 11
211552024-02-13T21:47:43.043Z08/30 Read block 593 len 12288
211562024-02-13T21:47:43.043Z09/30 Write block 557 len 8192 data: 19 15
211572024-02-13T21:47:43.043Z10/30 Flush
211582024-02-13T21:47:43.046Z11/30 Write block 422 len 40960 data: 13 15 12 12 14 15 17 21 16 16
211592024-02-13T21:47:43.049Z12/30 Flush
211602024-02-13T21:47:43.049Z13/30 Write block 55 len 32768 data: 17 17 17 19 23 20 22 21
211612024-02-13T21:47:43.051Z14/30 Read block 340 len 40960
211622024-02-13T21:47:43.051Z15/30 Write block 119 len 28672 data: 9 9 10 11 11 12 17
211632024-02-13T21:47:43.054Z16/30 Write block 237 len 24576 data: 10 15 15 16 16 17
211642024-02-13T21:47:43.054Z17/30 Read block 566 len 4096
211652024-02-13T21:47:43.054Z18/30 Write block 475 len 28672 data: 21 19 16 16 13 13 15
211662024-02-13T21:47:43.054Z19/30 Read block 457 len 20480
211672024-02-13T21:47:43.057Z20/30 Write block 154 len 24576 data: 15 16 14 14 13 12
211682024-02-13T21:47:43.057Z21/30 Read block 420 len 12288
211692024-02-13T21:47:43.057Z22/30 Read block 2 len 28672
211702024-02-13T21:47:43.063Z23/30 Write block 310 len 16384 data: 9 13 15 14
211712024-02-13T21:47:43.063Z24/30 Read block 330 len 16384
211722024-02-13T21:47:43.063Z25/30 Read block 591 len 28672
211732024-02-13T21:47:43.066Z26/30 Read block 60 len 40960
211742024-02-13T21:47:43.066Z27/30 Read block 130 len 12288
211752024-02-13T21:47:43.067Z28/30 Read block 107 len 8192
211762024-02-13T21:47:43.067Z29/30 Write block 221 len 24576 data: 19 22 23 21 19 15
211772024-02-13T21:47:43.067Z30/30 Read block 336 len 28672
211782024-02-13T21:47:43.067Z----------------------------------------------------------------
211792024-02-13T21:47:43.067Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:20
211802024-02-13T21:47:43.067ZStates: Active Active Active
211812024-02-13T21:47:43.067ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211822024-02-13T21:47:43.067Z 11 Acked 1010 Write 10 Done Done Done false
211832024-02-13T21:47:43.067Z 12 Acked 1011 Flush 0 Done Sent Done false
211842024-02-13T21:47:43.067Z 13 Acked 1012 Write 8 Done Sent Done false
211852024-02-13T21:47:43.067Z 14 Acked 1013 Read 10 Done Sent Done false
211862024-02-13T21:47:43.067Z 15 Acked 1014 Write 7 Done Sent Done false
211872024-02-13T21:47:43.067Z 16 Acked 1015 Write 6 Done Sent Done false
211882024-02-13T21:47:43.067Z 17 Acked 1016 Read 1 Done Sent Done false
211892024-02-13T21:47:43.067Z 18 Acked 1017 Write 7 Done Sent Done false
211902024-02-13T21:47:43.067Z 19 Acked 1018 Read 5 Done Sent Done false
211912024-02-13T21:47:43.067Z 20 Acked 1019 Write 6 Done Sent Done false
211922024-02-13T21:47:43.067Z 21 Acked 1020 Read 3 Done Sent Done false
211932024-02-13T21:47:43.067Z 22 Acked 1021 Read 7 Done Sent Done false
211942024-02-13T21:47:43.067Z 23 Acked 1022 Write 4 Done Sent Done false
211952024-02-13T21:47:43.067Z 24 Acked 1023 Read 4 Done Sent Done false
211962024-02-13T21:47:43.067Z 25 Acked 1024 Read 7 Done Sent Done false
211972024-02-13T21:47:43.067Z 26 Acked 1025 Read 10 Done Sent Done false
211982024-02-13T21:47:43.067Z 27 Acked 1026 Read 3 Done Sent Done false
211992024-02-13T21:47:43.068Z 28 Acked 1027 Read 2 Done Sent Done false
212002024-02-13T21:47:43.068Z 29 Acked 1028 Write 6 Done Sent Done false
212012024-02-13T21:47:43.068Z 30 Acked 1029 Read 7 Done Sent Sent false
212022024-02-13T21:47:43.068Z STATES DS:0 DS:1 DS:2 TOTAL
212032024-02-13T21:47:43.068Z New 0 0 0 0
212042024-02-13T21:47:43.068Z Sent 0 19 1 20
212052024-02-13T21:47:43.068Z Done 20 1 19 40
212062024-02-13T21:47:43.068Z Skipped 0 0 0 0
212072024-02-13T21:47:43.068Z Error 0 0 0 0
212082024-02-13T21:47:43.068ZLast Flush: 1011 1009 1011
212092024-02-13T21:47:43.068ZDownstairs last five completed: 1009 1008 1007 1006 1005
212102024-02-13T21:47:43.068ZUpstairs last five completed: 30 29 28 27 26
212112024-02-13T21:47:43.068Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:41.730141664Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1925}
212122024-02-13T21:47:43.068ZWrote out file "/tmp/test_repair_verify.data"
212132024-02-13T21:47:43.071Z
212142024-02-13T21:47:43.074ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
212152024-02-13T21:47:43.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212162024-02-13T21:47:43.091Z 0 000-019 192 192 192 344 344 344 F F F
212172024-02-13T21:47:43.091Z 1 020-039 195 195 195 347 347 347 F F F
212182024-02-13T21:47:43.091Z 2 040-059 194 194 194 345 345 345 T F T <---
212192024-02-13T21:47:43.091Z 3 060-079 192 192 192 344 344 344 T F T <---
212202024-02-13T21:47:43.091Z 4 080-099 194 194 194 345 345 345 F F F
212212024-02-13T21:47:43.091Z 5 100-119 192 192 192 344 344 344 T F T <---
212222024-02-13T21:47:43.092Z 6 120-139 178 178 178 326 326 326 T F T <---
212232024-02-13T21:47:43.092Z 7 140-159 195 195 195 347 347 347 T F T <---
212242024-02-13T21:47:43.092Z 8 160-179 187 187 187 340 340 340 F F F
212252024-02-13T21:47:43.095Z 9 180-199 188 188 188 341 341 341 F F F
212262024-02-13T21:47:43.095Z 10 200-219 196 196 196 348 348 348 F F F
212272024-02-13T21:47:43.095Z 11 220-239 192 192 192 344 344 344 T F T <---
212282024-02-13T21:47:43.095Z 12 240-259 192 192 192 344 344 344 T F T <---
212292024-02-13T21:47:43.095Z 13 260-279 195 195 195 347 347 347 F F F
212302024-02-13T21:47:43.095Z 14 280-299 186 186 186 338 338 338 F F F
212312024-02-13T21:47:43.095Z 15 300-319 186 186 186 338 338 338 T F T <---
212322024-02-13T21:47:43.095Z 16 320-339 191 191 191 343 343 343 F F F
212332024-02-13T21:47:43.095Z 17 340-359 192 192 192 344 344 344 F F F
212342024-02-13T21:47:43.095Z 18 360-379 194 194 194 345 345 345 F F F
212352024-02-13T21:47:43.095Z 19 380-399 196 196 196 348 348 348 F F F
212362024-02-13T21:47:43.095Z 20 400-419 196 196 196 348 348 348 F F F
212372024-02-13T21:47:43.095Z 21 420-439 196 192 196 349 344 349 F T F <---
212382024-02-13T21:47:43.095Z 22 440-459 192 192 192 344 344 344 F F F
212392024-02-13T21:47:43.095Z 23 460-479 192 192 192 344 344 344 T F T <---
212402024-02-13T21:47:43.095Z 24 480-499 192 192 192 344 344 344 T F T <---
212412024-02-13T21:47:43.095Z 25 500-519 195 195 195 347 347 347 F F F
212422024-02-13T21:47:43.095Z 26 520-539 195 195 195 347 347 347 F F F
212432024-02-13T21:47:43.096Z 27 540-559 196 196 196 348 348 348 F F F
212442024-02-13T21:47:43.096Z 28 560-579 194 194 194 345 345 345 F F F
212452024-02-13T21:47:43.096Z 29 580-599 191 191 191 343 343 343 F F F
212462024-02-13T21:47:43.096ZMax gen: 196, Max flush: 349
212472024-02-13T21:47:43.096ZError: Difference in extent metadata found!
212482024-02-13T21:47:43.096ZA Difference in extent metadata is expected here
212492024-02-13T21:47:43.096ZOn loop 98
212502024-02-13T21:47:43.096Z
212512024-02-13T21:47:43.099ZVerifying data now
212522024-02-13T21:47:47.680ZNo change after verify
212532024-02-13T21:47:47.680ZLoop: 98 Downstairs dump after verify (and repair):
212542024-02-13T21:47:47.695ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212552024-02-13T21:47:47.695Z 0 000-019 192 192 192 344 344 344 F F F
212562024-02-13T21:47:47.695Z 1 020-039 195 195 195 347 347 347 F F F
212572024-02-13T21:47:47.695Z 2 040-059 197 197 197 350 350 350 F F F
212582024-02-13T21:47:47.695Z 3 060-079 197 197 197 350 350 350 F F F
212592024-02-13T21:47:47.695Z 4 080-099 194 194 194 345 345 345 F F F
212602024-02-13T21:47:47.698Z 5 100-119 197 197 197 350 350 350 F F F
212612024-02-13T21:47:47.698Z 6 120-139 197 197 197 350 350 350 F F F
212622024-02-13T21:47:47.698Z 7 140-159 197 197 197 350 350 350 F F F
212632024-02-13T21:47:47.698Z 8 160-179 187 187 187 340 340 340 F F F
212642024-02-13T21:47:47.698Z 9 180-199 188 188 188 341 341 341 F F F
212652024-02-13T21:47:47.698Z 10 200-219 196 196 196 348 348 348 F F F
212662024-02-13T21:47:47.698Z 11 220-239 197 197 197 350 350 350 F F F
212672024-02-13T21:47:47.698Z 12 240-259 197 197 197 350 350 350 F F F
212682024-02-13T21:47:47.698Z 13 260-279 195 195 195 347 347 347 F F F
212692024-02-13T21:47:47.698Z 14 280-299 186 186 186 338 338 338 F F F
212702024-02-13T21:47:47.698Z 15 300-319 197 197 197 350 350 350 F F F
212712024-02-13T21:47:47.698Z 16 320-339 191 191 191 343 343 343 F F F
212722024-02-13T21:47:47.698Z 17 340-359 192 192 192 344 344 344 F F F
212732024-02-13T21:47:47.698Z 18 360-379 194 194 194 345 345 345 F F F
212742024-02-13T21:47:47.698Z 19 380-399 196 196 196 348 348 348 F F F
212752024-02-13T21:47:47.698Z 20 400-419 196 196 196 348 348 348 F F F
212762024-02-13T21:47:47.699Z 21 420-439 196 196 196 349 349 349 F F F
212772024-02-13T21:47:47.699Z 22 440-459 192 192 192 344 344 344 F F F
212782024-02-13T21:47:47.699Z 23 460-479 197 197 197 350 350 350 F F F
212792024-02-13T21:47:47.699Z 24 480-499 197 197 197 350 350 350 F F F
212802024-02-13T21:47:47.699Z 25 500-519 195 195 195 347 347 347 F F F
212812024-02-13T21:47:47.699Z 26 520-539 195 195 195 347 347 347 F F F
212822024-02-13T21:47:47.699Z 27 540-559 196 196 196 348 348 348 F F F
212832024-02-13T21:47:47.699Z 28 560-579 194 194 194 345 345 345 F F F
212842024-02-13T21:47:47.699Z 29 580-599 191 191 191 343 343 343 F F F
212852024-02-13T21:47:47.699ZMax gen: 197, Max flush: 350
212862024-02-13T21:47:47.699Z
212872024-02-13T21:47:47.699ZBegin loop 99
212882024-02-13T21:47:47.699ZDownstairs to restart: 2
212892024-02-13T21:47:47.715Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378431831Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
212902024-02-13T21:47:47.715Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"29185f5538632a13039785e911d1aaa0146bfa14\",\n git_commit_timestamp: \"2024-02-13T21:13:12.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378651382Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
212912024-02-13T21:47:47.715Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378673294Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
212922024-02-13T21:47:47.720Z{"msg":"Crucible 8e3ab4e8-c0bb-4846-975e-d4443514d504 has session id: 604858e6-5ccf-40c4-82e9-b19f55c63ff5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378708411Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
212932024-02-13T21:47:47.720ZCrucible runtime is spawned
212942024-02-13T21:47:47.720Z{"msg":"Upstairs opts: Upstairs UUID: 8e3ab4e8-c0bb-4846-975e-d4443514d504, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378764501Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
212952024-02-13T21:47:47.720Z{"msg":"Crucible stats registered with UUID: 8e3ab4e8-c0bb-4846-975e-d4443514d504","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378799848Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
212962024-02-13T21:47:47.720Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378835835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
212972024-02-13T21:47:47.720Z{"msg":"8e3ab4e8-c0bb-4846-975e-d4443514d504 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.378964608Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
212982024-02-13T21:47:47.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379005723Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
212992024-02-13T21:47:47.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379033253Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213002024-02-13T21:47:47.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.37907101Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213012024-02-13T21:47:47.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.37909846Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213022024-02-13T21:47:47.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379131557Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213032024-02-13T21:47:47.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379155109Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213042024-02-13T21:47:47.721Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379190876Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213052024-02-13T21:47:47.721Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379224234Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213062024-02-13T21:47:47.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379269367Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213072024-02-13T21:47:47.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379355346Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213082024-02-13T21:47:47.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379426121Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213092024-02-13T21:47:47.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.379473273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"io task","client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213102024-02-13T21:47:47.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380159105Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213112024-02-13T21:47:47.721Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380218273Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213122024-02-13T21:47:47.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380274043Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213132024-02-13T21:47:47.721Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380343968Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213142024-02-13T21:47:47.721Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380863639Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213152024-02-13T21:47:47.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380947399Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213162024-02-13T21:47:47.721Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380973349Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213172024-02-13T21:47:47.721Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.380989754Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213182024-02-13T21:47:47.721Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381569753Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213192024-02-13T21:47:47.721Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381612318Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213202024-02-13T21:47:47.721Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381627672Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213212024-02-13T21:47:47.721Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381647285Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213222024-02-13T21:47:47.721Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.38166113Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213232024-02-13T21:47:47.721Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381684601Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213242024-02-13T21:47:47.721Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.381734173Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213252024-02-13T21:47:47.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382087835Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213262024-02-13T21:47:47.725Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.38212955Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213272024-02-13T21:47:47.725Z{"msg":"[0]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 350, 340, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382145864Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213282024-02-13T21:47:47.725Z{"msg":"[0]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 197, 187, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382168126Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213292024-02-13T21:47:47.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.38218319Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213302024-02-13T21:47:47.725Z{"msg":"[1]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 350, 340, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382203953Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213312024-02-13T21:47:47.725Z{"msg":"[1]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 197, 187, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382218188Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213322024-02-13T21:47:47.726Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382243159Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213332024-02-13T21:47:47.726Z{"msg":"Wait for a query_work_queue command to finish before sending IO
213342024-02-13T21:47:47.726Z[2]R flush_numbers[0..12]: [344, 347, 350, 350, 345, 350, 350, 350, 340, 341, 348, 350]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382280575Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213352024-02-13T21:47:47.726Z{"msg":"[2]R generation[0..12]: [192, 195, 197, 197, 194, 197, 197, 197, 187, 188, 196, 197]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382318592Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213362024-02-13T21:47:47.726Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382344362Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213372024-02-13T21:47:47.726Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382358267Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213382024-02-13T21:47:47.726Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382388456Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213392024-02-13T21:47:47.726Z{"msg":"Next flush: 351","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382425153Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213402024-02-13T21:47:47.726Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382483761Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"":"downstairs","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213412024-02-13T21:47:47.726Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382508772Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213422024-02-13T21:47:47.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382529405Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"0","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213432024-02-13T21:47:47.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382561553Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"1","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213442024-02-13T21:47:47.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382585704Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"client":"2","":"downstairs","session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213452024-02-13T21:47:47.726Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382609256Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213462024-02-13T21:47:47.726Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382637696Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213472024-02-13T21:47:47.726Z{"msg":"8e3ab4e8-c0bb-4846-975e-d4443514d504 is now active with session: 604858e6-5ccf-40c4-82e9-b19f55c63ff5","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382667945Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213482024-02-13T21:47:47.726Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213492024-02-13T21:47:47.726Z,"time":"2024-02-13T21:47:46.382686138Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933,"session_id":"604858e6-5ccf-40c4-82e9-b19f55c63ff5"}
213502024-02-13T21:47:47.726Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-13T21:47:46.382704032Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
213512024-02-13T21:47:47.726ZLoading write count information from file "/tmp/test_repair_verify.data"
213522024-02-13T21:47:47.726ZRun Repair workload
213532024-02-13T21:47:47.726Z01/30 Write block 148 len 20480 data: 13 14 11 11 11
213542024-02-13T21:47:47.726Z02/30 Read block 195 len 40960
213552024-02-13T21:47:47.726Z03/30 Read block 420 len 32768
213562024-02-13T21:47:47.726Z04/30 Write block 170 len 12288 data: 7 8 12
213572024-02-13T21:47:47.730Z05/30 Write block 424 len 28672 data: 13 13 15 16 18 22 17
213582024-02-13T21:47:47.730Z06/30 Read block 386 len 32768
213592024-02-13T21:47:47.730Z07/30 Write block 558 len 32768 data: 16 11 13 16 16 15 17 16
213602024-02-13T21:47:47.730Z08/30 Write block 337 len 8192 data: 13 12
213612024-02-13T21:47:47.730Z09/30 Read block 70 len 8192
213622024-02-13T21:47:47.730Z10/30 Read block 440 len 32768
213632024-02-13T21:47:47.730Z11/30 Read block 223 len 8192
213642024-02-13T21:47:47.733Z12/30 Read block 415 len 32768
213652024-02-13T21:47:47.733Z13/30 Read block 302 len 16384
213662024-02-13T21:47:47.733Z14/30 Read block 150 len 8192
213672024-02-13T21:47:47.733Z15/30 Read block 467 len 12288
213682024-02-13T21:47:47.736Z16/30 Write block 435 len 24576 data: 14 9 9 12 12 11
213692024-02-13T21:47:47.736Z17/30 Flush
213702024-02-13T21:47:47.739Z18/30 Write block 335 len 16384 data: 18 16 14 13
213712024-02-13T21:47:47.739Z19/30 Read block 490 len 4096
213722024-02-13T21:47:47.739Z20/30 Read block 183 len 20480
213732024-02-13T21:47:47.742Z21/30 Write block 580 len 32768 data: 9 9 11 13 14 15 12 12
213742024-02-13T21:47:47.742Z22/30 Read block 570 len 12288
213752024-02-13T21:47:47.742Z23/30 Flush
213762024-02-13T21:47:47.745Z24/30 Write block 331 len 28672 data: 22 21 18 14 19 17 15
213772024-02-13T21:47:47.745Z25/30 Write block 339 len 16384 data: 11 14 14 15
213782024-02-13T21:47:47.751Z26/30 Write block 101 len 28672 data: 12 18 16 14 14 12 12
213792024-02-13T21:47:47.751Z27/30 Write block 585 len 16384 data: 16 13 13 11
213802024-02-13T21:47:47.751Z28/30 Write block 446 len 4096 data: 16
213812024-02-13T21:47:47.751Z29/30 Write block 185 len 20480 data: 18 15 14 16 15
213822024-02-13T21:47:47.751Z30/30 Read block 49 len 16384
213832024-02-13T21:47:47.754Z----------------------------------------------------------------
213842024-02-13T21:47:47.754Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213852024-02-13T21:47:47.754ZStates: Active Active Active
213862024-02-13T21:47:47.754ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213872024-02-13T21:47:47.754Z 1 Acked 1000 Write 5 Done Done Done false
213882024-02-13T21:47:47.754Z 2 Acked 1001 Read 10 Done Done Done false
213892024-02-13T21:47:47.755Z 3 Acked 1002 Read 8 Done Done Done false
213902024-02-13T21:47:47.755Z 4 Acked 1003 Write 3 Done Done Done false
213912024-02-13T21:47:47.755Z 5 Acked 1004 Write 7 Done Done Done false
213922024-02-13T21:47:47.755Z 6 Acked 1005 Read 8 Done Done Done false
213932024-02-13T21:47:47.755Z 7 Acked 1006 Write 8 Done Done Sent false
213942024-02-13T21:47:47.755Z 8 Acked 1007 Write 2 Done Done Sent false
213952024-02-13T21:47:47.755Z 9 Acked 1008 Read 2 Done Done Sent false
213962024-02-13T21:47:47.755Z 10 Acked 1009 Read 8 Done Done Sent false
213972024-02-13T21:47:47.755Z 11 Acked 1010 Read 2 Done Done Sent false
213982024-02-13T21:47:47.755Z 12 Acked 1011 Read 8 Done Done Sent false
213992024-02-13T21:47:47.755Z 13 Acked 1012 Read 4 Done Done Sent false
214002024-02-13T21:47:47.755Z 14 Acked 1013 Read 2 Done Done Sent false
214012024-02-13T21:47:47.755Z 15 Acked 1014 Read 3 Done Done Sent false
214022024-02-13T21:47:47.755Z 16 Acked 1015 Write 6 Done Done Sent false
214032024-02-13T21:47:47.755Z 17 Acked 1016 Flush 0 Done Done Sent false
214042024-02-13T21:47:47.755Z 18 Acked 1017 Write 4 Done Done Sent false
214052024-02-13T21:47:47.755Z 19 Acked 1018 Read 1 Done Done Sent false
214062024-02-13T21:47:47.755Z 20 Acked 1019 Read 5 Done Done Sent false
214072024-02-13T21:47:47.755Z 21 Acked 1020 Write 8 Done Done Sent false
214082024-02-13T21:47:47.755Z 22 Acked 1021 Read 3 Done Done Sent false
214092024-02-13T21:47:47.755Z 23 Acked 1022 Flush 0 Done Done Sent false
214102024-02-13T21:47:47.755Z 24 Acked 1023 Write 7 Done Done Sent false
214112024-02-13T21:47:47.755Z 25 Acked 1024 Write 4 Done Done Sent false
214122024-02-13T21:47:47.755Z 26 Acked 1025 Write 7 Done Done Sent false
214132024-02-13T21:47:47.755Z 27 Acked 1026 Write 4 Done Done Sent false
214142024-02-13T21:47:47.756Z 28 Acked 1027 Write 1 Done Done Sent false
214152024-02-13T21:47:47.756Z 29 Acked 1028 Write 5 Done Done Sent false
214162024-02-13T21:47:47.756Z 30 Acked 1029 Read 4 Sent Done Sent false
214172024-02-13T21:47:47.756Z STATES DS:0 DS:1 DS:2 TOTAL
214182024-02-13T21:47:47.756Z New 0 0 0 0
214192024-02-13T21:47:47.756Z Sent 1 0 24 25
214202024-02-13T21:47:47.756Z Done 29 30 6 65
214212024-02-13T21:47:47.756Z Skipped 0 0 0 0
214222024-02-13T21:47:47.756Z Error 0 0 0 0
214232024-02-13T21:47:47.756ZLast Flush: 1022 1022 0
214242024-02-13T21:47:47.756ZDownstairs last five completed:
214252024-02-13T21:47:47.756ZUpstairs last five completed: 30 29 28 27 26
214262024-02-13T21:47:47.756Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-13T21:47:46.412175681Z","hostname":"ip-10-150-1-148.us-west-2.compute.internal","pid":1933}
214272024-02-13T21:47:47.756ZWrote out file "/tmp/test_repair_verify.data"
214282024-02-13T21:47:47.756Z
214292024-02-13T21:47:47.759ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214302024-02-13T21:47:47.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214312024-02-13T21:47:47.773Z 0 000-019 192 192 192 344 344 344 F F F
214322024-02-13T21:47:47.773Z 1 020-039 195 195 195 347 347 347 F F F
214332024-02-13T21:47:47.774Z 2 040-059 197 197 197 350 350 350 F F F
214342024-02-13T21:47:47.774Z 3 060-079 197 197 197 350 350 350 F F F
214352024-02-13T21:47:47.774Z 4 080-099 194 194 194 345 345 345 F F F
214362024-02-13T21:47:47.774Z 5 100-119 197 197 197 350 350 350 T T F <---
214372024-02-13T21:47:47.774Z 6 120-139 197 197 197 350 350 350 F F F
214382024-02-13T21:47:47.774Z 7 140-159 198 198 197 351 351 350 F F T <---
214392024-02-13T21:47:47.774Z 8 160-179 198 198 187 351 351 340 F F T <---
214402024-02-13T21:47:47.774Z 9 180-199 188 188 188 341 341 341 T T F <---
214412024-02-13T21:47:47.774Z 10 200-219 196 196 196 348 348 348 F F F
214422024-02-13T21:47:47.774Z 11 220-239 197 197 197 350 350 350 F F F
214432024-02-13T21:47:47.774Z 12 240-259 197 197 197 350 350 350 F F F
214442024-02-13T21:47:47.786Z 13 260-279 195 195 195 347 347 347 F F F
214452024-02-13T21:47:47.786Z 14 280-299 186 186 186 338 338 338 F F F
214462024-02-13T21:47:47.786Z 15 300-319 197 197 197 350 350 350 F F F
214472024-02-13T21:47:47.786Z 16 320-339 198 198 191 352 352 343 T T F <---
214482024-02-13T21:47:47.786Z 17 340-359 192 192 192 344 344 344 T T F <---
214492024-02-13T21:47:47.786Z 18 360-379 194 194 194 345 345 345 F F F
214502024-02-13T21:47:47.786Z 19 380-399 196 196 196 348 348 348 F F F
214512024-02-13T21:47:47.786Z 20 400-419 196 196 196 348 348 348 F F F
214522024-02-13T21:47:47.787Z 21 420-439 198 198 196 351 351 349 F F T <---
214532024-02-13T21:47:47.787Z 22 440-459 198 198 192 351 351 344 T T F <---
214542024-02-13T21:47:47.787Z 23 460-479 197 197 197 350 350 350 F F F
214552024-02-13T21:47:47.787Z 24 480-499 197 197 197 350 350 350 F F F
214562024-02-13T21:47:47.787Z 25 500-519 195 195 195 347 347 347 F F F
214572024-02-13T21:47:47.787Z 26 520-539 195 195 195 347 347 347 F F F
214582024-02-13T21:47:47.787Z 27 540-559 198 198 196 351 351 348 F F F <---
214592024-02-13T21:47:47.787Z 28 560-579 198 198 194 351 351 345 F F F <---
214602024-02-13T21:47:47.787Z 29 580-599 198 198 191 352 352 343 T T F <---
214612024-02-13T21:47:47.787ZMax gen: 198, Max flush: 352
214622024-02-13T21:47:47.787ZError: Difference in extent metadata found!
214632024-02-13T21:47:47.787ZA Difference in extent metadata is expected here
214642024-02-13T21:47:47.787ZOn loop 99
214652024-02-13T21:47:47.787Z
214662024-02-13T21:47:47.787ZVerifying data now
214672024-02-13T21:47:53.334ZNo change after verify
214682024-02-13T21:47:53.334ZLoop: 99 Downstairs dump after verify (and repair):
214692024-02-13T21:47:53.352ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214702024-02-13T21:47:53.352Z 0 000-019 192 192 192 344 344 344 F F F
214712024-02-13T21:47:53.352Z 1 020-039 195 195 195 347 347 347 F F F
214722024-02-13T21:47:53.352Z 2 040-059 197 197 197 350 350 350 F F F
214732024-02-13T21:47:53.352Z 3 060-079 197 197 197 350 350 350 F F F
214742024-02-13T21:47:53.352Z 4 080-099 194 194 194 345 345 345 F F F
214752024-02-13T21:47:53.352Z 5 100-119 199 199 199 353 353 353 F F F
214762024-02-13T21:47:53.352Z 6 120-139 197 197 197 350 350 350 F F F
214772024-02-13T21:47:53.353Z 7 140-159 198 198 198 351 351 351 F F F
214782024-02-13T21:47:53.353Z 8 160-179 198 198 198 351 351 351 F F F
214792024-02-13T21:47:53.353Z 9 180-199 199 199 199 353 353 353 F F F
214802024-02-13T21:47:53.353Z 10 200-219 196 196 196 348 348 348 F F F
214812024-02-13T21:47:53.353Z 11 220-239 197 197 197 350 350 350 F F F
214822024-02-13T21:47:53.353Z 12 240-259 197 197 197 350 350 350 F F F
214832024-02-13T21:47:53.353Z 13 260-279 195 195 195 347 347 347 F F F
214842024-02-13T21:47:53.356Z 14 280-299 186 186 186 338 338 338 F F F
214852024-02-13T21:47:53.356Z 15 300-319 197 197 197 350 350 350 F F F
214862024-02-13T21:47:53.356Z 16 320-339 199 199 199 353 353 353 F F F
214872024-02-13T21:47:53.356Z 17 340-359 199 199 199 353 353 353 F F F
214882024-02-13T21:47:53.356Z 18 360-379 194 194 194 345 345 345 F F F
214892024-02-13T21:47:53.356Z 19 380-399 196 196 196 348 348 348 F F F
214902024-02-13T21:47:53.356Z 20 400-419 196 196 196 348 348 348 F F F
214912024-02-13T21:47:53.356Z 21 420-439 198 198 198 351 351 351 F F F
214922024-02-13T21:47:53.356Z 22 440-459 199 199 199 353 353 353 F F F
214932024-02-13T21:47:53.356Z 23 460-479 197 197 197 350 350 350 F F F
214942024-02-13T21:47:53.356Z 24 480-499 197 197 197 350 350 350 F F F
214952024-02-13T21:47:53.357Z 25 500-519 195 195 195 347 347 347 F F F
214962024-02-13T21:47:53.357Z 26 520-539 195 195 195 347 347 347 F F F
214972024-02-13T21:47:53.357Z 27 540-559 198 198 198 351 351 351 F F F
214982024-02-13T21:47:53.357Z 28 560-579 198 198 198 351 351 351 F F F
214992024-02-13T21:47:53.357Z 29 580-599 199 199 199 353 353 353 F F F
215002024-02-13T21:47:53.357ZMax gen: 199, Max flush: 353
215012024-02-13T21:47:53.357Z8:35 Test duration
215022024-02-13T21:47:53.357ZTest completed
215032024-02-13T21:47:53.357Z
215042024-02-13T21:47:53.357Zreal 8:34.313350771
215052024-02-13T21:47:53.357Zuser 1:37.514423135
215062024-02-13T21:47:53.357Zsys 17.513356335
215072024-02-13T21:47:53.357Ztrap 0.087307883
215082024-02-13T21:47:53.357Ztflt 0.055299997
215092024-02-13T21:47:53.357Zdflt 0.063596912
215102024-02-13T21:47:53.357Zkflt 0.002882473
215112024-02-13T21:47:53.357Zlock 10:00:20.359067001
215122024-02-13T21:47:53.357Zslp 40:48.159014458
215132024-02-13T21:47:53.357Zlat 6.908365183
215142024-02-13T21:47:53.357Zstop 0.047541946
215152024-02-13T21:47:53.357Zprocess exited: duration 523408 ms, exit code 0
 
215162024-02-13T21:47:53.371Zfound 4 output files
215172024-02-13T21:47:53.371Zuploading: /tmp/test_repair_ds8810.txt (166752 bytes)
215182024-02-13T21:47:54.388Zuploaded: /tmp/test_repair_ds8810.txt
215192024-02-13T21:47:54.388Zuploading: /tmp/test_repair_ds8820.txt (30677 bytes)
215202024-02-13T21:47:55.401Zuploaded: /tmp/test_repair_ds8820.txt
215212024-02-13T21:47:55.402Zuploading: /tmp/test_repair_ds8830.txt (18507 bytes)
215222024-02-13T21:47:56.425Zuploaded: /tmp/test_repair_ds8830.txt
215232024-02-13T21:47:56.425Zuploading: /tmp/test_repair_out.txt (78743 bytes)
215242024-02-13T21:47:57.440Zuploaded: /tmp/test_repair_out.txt